{ "architecture": "block", "artifacts": { "grok_review_sha256": "2da1b6bf1f455c4121a7a2c5cfe40e102327dabafc7e24dccc23ed0d00ac6d71", "protocol_sha256": "5ecc7ca92314ddb50aecf0cb50e115814c8983aa8bffb30e3634f7b3ce6dca1d", "runner_sha256": "b42879e242a2f2d54aa6a87a718aeac4cf4509eae42da2b14403656675a8b03d", "scoping_sha256": "670ca4edf31a4be1f54937d9c7a760dba7a96e1e820c38c6b10405e22b078fc8" }, "batch_size": 32, "canonical_sha256_without_self": "f0a44f119836ed632c15880c3c2bb225173c0a05c50ea07abbe0e464ff407592", "depth": 32, "diagnostics": [ { "endpoint_exactness": { "checks": { "detached_learned_round06_exact": true, "uniform_all_round06_exact": true }, "passed": true }, "forward_identity_gate": { "comparisons": { "uniform_all": { "activations_exact": true, "logits_exact": true, "loss_exact": true, "mixer_summaries_exact": true }, "uniform_depth_all": { "activations_exact": true, "logits_exact": true, "loss_exact": true, "mixer_summaries_exact": true }, "uniform_except_group_6": { "activations_exact": true, "logits_exact": true, "loss_exact": true, "mixer_summaries_exact": true }, "uniform_except_group_7": { "activations_exact": true, "logits_exact": true, "loss_exact": true, "mixer_summaries_exact": true }, "uniform_except_groups_6_7": { "activations_exact": true, "logits_exact": true, "loss_exact": true, "mixer_summaries_exact": true }, "uniform_group_6_attention_only": { "activations_exact": true, "logits_exact": true, "loss_exact": true, "mixer_summaries_exact": true }, "uniform_group_6_mlp_only": { "activations_exact": true, "logits_exact": true, "loss_exact": true, "mixer_summaries_exact": true }, "uniform_group_6_only": { "activations_exact": true, "logits_exact": true, "loss_exact": true, "mixer_summaries_exact": true }, "uniform_group_7_attention_only": { "activations_exact": true, "logits_exact": true, "loss_exact": true, "mixer_summaries_exact": true }, "uniform_group_7_mlp_only": { "activations_exact": true, "logits_exact": true, "loss_exact": true, "mixer_summaries_exact": true }, "uniform_group_7_only": { "activations_exact": true, "logits_exact": true, "loss_exact": true, "mixer_summaries_exact": true }, "uniform_groups_6_7_only": { "activations_exact": true, "logits_exact": true, "loss_exact": true, "mixer_summaries_exact": true }, "uniform_output_only": { "activations_exact": true, "logits_exact": true, "loss_exact": true, "mixer_summaries_exact": true } }, "passed": true }, "initialization_negative_control": { "comparisons": { "parent_learned_vs_detached": { "checks": { "attention_branch_output": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true }, "mlp_branch_output": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true }, "post_attention_state": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true }, "post_mlp_state": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true }, "pre_attention_input": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true }, "pre_mlp_input": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true } }, "passed": true }, "uniform_all": { "checks": { "attention_branch_output": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true }, "mlp_branch_output": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true }, "post_attention_state": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true }, "post_mlp_state": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true }, "pre_attention_input": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true }, "pre_mlp_input": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true } }, "passed": true }, "uniform_depth_all": { "checks": { "attention_branch_output": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true }, "mlp_branch_output": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true }, "post_attention_state": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true }, "post_mlp_state": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true }, "pre_attention_input": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true }, "pre_mlp_input": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true } }, "passed": true }, "uniform_except_group_6": { "checks": { "attention_branch_output": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true }, "mlp_branch_output": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true }, "post_attention_state": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true }, "post_mlp_state": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true }, "pre_attention_input": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true }, "pre_mlp_input": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true } }, "passed": true }, "uniform_except_group_7": { "checks": { "attention_branch_output": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true }, "mlp_branch_output": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true }, "post_attention_state": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true }, "post_mlp_state": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true }, "pre_attention_input": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true }, "pre_mlp_input": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true } }, "passed": true }, "uniform_except_groups_6_7": { "checks": { "attention_branch_output": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true }, "mlp_branch_output": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true }, "post_attention_state": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true }, "post_mlp_state": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true }, "pre_attention_input": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true }, "pre_mlp_input": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true } }, "passed": true }, "uniform_group_6_attention_only": { "checks": { "attention_branch_output": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true }, "mlp_branch_output": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true }, "post_attention_state": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true }, "post_mlp_state": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true }, "pre_attention_input": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true }, "pre_mlp_input": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true } }, "passed": true }, "uniform_group_6_mlp_only": { "checks": { "attention_branch_output": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true }, "mlp_branch_output": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true }, "post_attention_state": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true }, "post_mlp_state": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true }, "pre_attention_input": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true }, "pre_mlp_input": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true } }, "passed": true }, "uniform_group_6_only": { "checks": { "attention_branch_output": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true }, "mlp_branch_output": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true }, "post_attention_state": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true }, "post_mlp_state": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true }, "pre_attention_input": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true }, "pre_mlp_input": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true } }, "passed": true }, "uniform_group_7_attention_only": { "checks": { "attention_branch_output": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true }, "mlp_branch_output": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true }, "post_attention_state": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true }, "post_mlp_state": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true }, "pre_attention_input": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true }, "pre_mlp_input": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true } }, "passed": true }, "uniform_group_7_mlp_only": { "checks": { "attention_branch_output": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true }, "mlp_branch_output": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true }, "post_attention_state": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true }, "post_mlp_state": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true }, "pre_attention_input": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true }, "pre_mlp_input": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true } }, "passed": true }, "uniform_group_7_only": { "checks": { "attention_branch_output": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true }, "mlp_branch_output": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true }, "post_attention_state": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true }, "post_mlp_state": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true }, "pre_attention_input": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true }, "pre_mlp_input": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true } }, "passed": true }, "uniform_groups_6_7_only": { "checks": { "attention_branch_output": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true }, "mlp_branch_output": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true }, "post_attention_state": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true }, "post_mlp_state": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true }, "pre_attention_input": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true }, "pre_mlp_input": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true } }, "passed": true }, "uniform_output_only": { "checks": { "attention_branch_output": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true }, "mlp_branch_output": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true }, "post_attention_state": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true }, "post_mlp_state": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true }, "pre_attention_input": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true }, "pre_mlp_input": { "max_normalized_absolute_error": 0.0, "max_raw_relative_error": 0.0, "passed": true } }, "passed": true } }, "passed": true }, "local_matrix": { "detached_learned": { "bits_per_byte": 7.959341947457508, "forward": { "activation_sha256": { "attention_branch_output": "676759a2d7c0615ad7e1f246e90927fb829d00c8b6c3745032ae6b27ab7d902a", "mlp_branch_output": "afe29f3ef4d2992353f0bb59fe7c5c2c5050fca671a53184825a8d63cd83a2bc", "post_attention_state": "31853ffb07b24cc59e774bb20656840b4358b3080b42516d380f2dc0f46404c8", "post_mlp_state": "7fd9c911e93175896ad618842c120e77b3f017c17b7eec9c2bfb01fc6dbc8ac2", "pre_attention_input": "de71d4cf2d509a1063f6eac7f50c4885e9a058dee7e4b993ad40f5527f528d00", "pre_mlp_input": "2999a257265024350fd2665e2a8d452a03cd662193b4245d15290408ad29922d" }, "logits_sha256": "873a309a197bfd08fa523edcb440d2697903a34abc02dad1c7f5991509c3c151", "loss_nats": 5.516995429992676, "mixer_summary_sha256": "8450b24bdc777885a3357dc7b8080e7410acbe96943686433da76f379295133a" }, "loss_nats": 5.516995429992676, "loss_scale": 1.0, "mixers": [ { "branch": "attention", "entropy_mean": -0.0, "group": 1, "latest_source_mass": 1.0, "layer": 1, "max_source_mass": 1.0, "mixer_index": 0, "normalized_entropy": 1.0, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 1.0, "median": 1.0, "p05": 1.0, "p95": 1.0 } ], "sources": 1 }, { "branch": "mlp", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 1, "max_source_mass": 0.5, "mixer_index": 1, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 2, "max_source_mass": 0.5, "mixer_index": 2, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 2, "max_source_mass": 0.5, "mixer_index": 3, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 3, "max_source_mass": 0.5, "mixer_index": 4, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 3, "max_source_mass": 0.5, "mixer_index": 5, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 4, "max_source_mass": 0.5, "mixer_index": 6, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 4, "max_source_mass": 0.5, "mixer_index": 7, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6931471824645996, "group": 2, "latest_source_mass": 0.5, "layer": 5, "max_source_mass": 0.5, "mixer_index": 8, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "completed_group_1", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 5, "max_source_mass": 0.3333333432674408, "mixer_index": 9, "normalized_entropy": 1.0000000180537645, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 6, "max_source_mass": 0.3333333432674408, "mixer_index": 10, "normalized_entropy": 1.0000000180537645, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 6, "max_source_mass": 0.3333333432674408, "mixer_index": 11, "normalized_entropy": 1.0000000180537645, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 7, "max_source_mass": 0.3333333432674408, "mixer_index": 12, "normalized_entropy": 1.0000000180537645, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 7, "max_source_mass": 0.3333333432674408, "mixer_index": 13, "normalized_entropy": 1.0000000180537645, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 8, "max_source_mass": 0.3333333432674408, "mixer_index": 14, "normalized_entropy": 1.0000000180537645, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 8, "max_source_mass": 0.3333333432674408, "mixer_index": 15, "normalized_entropy": 1.0000000180537645, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0986123085021973, "group": 3, "latest_source_mass": 0.3333333432674408, "layer": 9, "max_source_mass": 0.3333333432674408, "mixer_index": 16, "normalized_entropy": 1.0000000180537645, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_2", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 9, "max_source_mass": 0.25, "mixer_index": 17, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 10, "max_source_mass": 0.25, "mixer_index": 18, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 10, "max_source_mass": 0.25, "mixer_index": 19, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 11, "max_source_mass": 0.25, "mixer_index": 20, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 11, "max_source_mass": 0.25, "mixer_index": 21, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 12, "max_source_mass": 0.25, "mixer_index": 22, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 12, "max_source_mass": 0.25, "mixer_index": 23, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3862943649291992, "group": 4, "latest_source_mass": 0.25, "layer": 13, "max_source_mass": 0.25, "mixer_index": 24, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_3", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 13, "max_source_mass": 0.20000000298023224, "mixer_index": 25, "normalized_entropy": 1.0000000186840277, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 14, "max_source_mass": 0.20000000298023224, "mixer_index": 26, "normalized_entropy": 1.0000000186840277, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 14, "max_source_mass": 0.20000000298023224, "mixer_index": 27, "normalized_entropy": 1.0000000186840277, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 15, "max_source_mass": 0.20000000298023224, "mixer_index": 28, "normalized_entropy": 1.0000000186840277, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 15, "max_source_mass": 0.20000000298023224, "mixer_index": 29, "normalized_entropy": 1.0000000186840277, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 16, "max_source_mass": 0.20000000298023224, "mixer_index": 30, "normalized_entropy": 1.0000000186840277, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 16, "max_source_mass": 0.20000000298023224, "mixer_index": 31, "normalized_entropy": 1.0000000186840277, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.6094379425048828, "group": 5, "latest_source_mass": 0.20000000298023224, "layer": 17, "max_source_mass": 0.20000000298023224, "mixer_index": 32, "normalized_entropy": 1.0000000186840277, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_4", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 17, "max_source_mass": 0.1666666716337204, "mixer_index": 33, "normalized_entropy": 1.0000001451965654, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 18, "max_source_mass": 0.1666666716337204, "mixer_index": 34, "normalized_entropy": 1.0000001451965654, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 18, "max_source_mass": 0.1666666716337204, "mixer_index": 35, "normalized_entropy": 1.0000001451965654, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 19, "max_source_mass": 0.1666666716337204, "mixer_index": 36, "normalized_entropy": 1.0000001451965654, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 19, "max_source_mass": 0.1666666716337204, "mixer_index": 37, "normalized_entropy": 1.0000001451965654, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 20, "max_source_mass": 0.1666666716337204, "mixer_index": 38, "normalized_entropy": 1.0000001451965654, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 20, "max_source_mass": 0.1666666716337204, "mixer_index": 39, "normalized_entropy": 1.0000001451965654, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.791759729385376, "group": 6, "latest_source_mass": 0.1666666716337204, "layer": 21, "max_source_mass": 0.1666666716337204, "mixer_index": 40, "normalized_entropy": 1.0000001451965654, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_5", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 21, "max_source_mass": 0.1428571492433548, "mixer_index": 41, "normalized_entropy": 1.00000003408302, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 22, "max_source_mass": 0.1428571492433548, "mixer_index": 42, "normalized_entropy": 1.00000003408302, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 22, "max_source_mass": 0.1428571492433548, "mixer_index": 43, "normalized_entropy": 1.00000003408302, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 23, "max_source_mass": 0.1428571492433548, "mixer_index": 44, "normalized_entropy": 1.00000003408302, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 23, "max_source_mass": 0.1428571492433548, "mixer_index": 45, "normalized_entropy": 1.00000003408302, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 24, "max_source_mass": 0.1428571492433548, "mixer_index": 46, "normalized_entropy": 1.00000003408302, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 24, "max_source_mass": 0.1428571492433548, "mixer_index": 47, "normalized_entropy": 1.00000003408302, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9459102153778076, "group": 7, "latest_source_mass": 0.1428571492433548, "layer": 25, "max_source_mass": 0.1428571492433548, "mixer_index": 48, "normalized_entropy": 1.00000003408302, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_6", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 25, "max_source_mass": 0.125, "mixer_index": 49, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 26, "max_source_mass": 0.125, "mixer_index": 50, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 26, "max_source_mass": 0.125, "mixer_index": 51, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 27, "max_source_mass": 0.125, "mixer_index": 52, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 27, "max_source_mass": 0.125, "mixer_index": 53, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 28, "max_source_mass": 0.125, "mixer_index": 54, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 28, "max_source_mass": 0.125, "mixer_index": 55, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.079441547393799, "group": 8, "latest_source_mass": 0.125, "layer": 29, "max_source_mass": 0.125, "mixer_index": 56, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_7", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 29, "max_source_mass": 0.1111111119389534, "mixer_index": 57, "normalized_entropy": 1.0000000180537645, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 30, "max_source_mass": 0.1111111119389534, "mixer_index": 58, "normalized_entropy": 1.0000000180537645, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 30, "max_source_mass": 0.1111111119389534, "mixer_index": 59, "normalized_entropy": 1.0000000180537645, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 31, "max_source_mass": 0.1111111119389534, "mixer_index": 60, "normalized_entropy": 1.0000000180537645, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 31, "max_source_mass": 0.1111111119389534, "mixer_index": 61, "normalized_entropy": 1.0000000180537645, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 32, "max_source_mass": 0.1111111119389534, "mixer_index": 62, "normalized_entropy": 1.0000000180537645, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 32, "max_source_mass": 0.1111111119389534, "mixer_index": 63, "normalized_entropy": 1.0000000180537645, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "output", "entropy_mean": 2.1972246170043945, "group": null, "latest_source_mass": 0.1111111119389534, "layer": null, "max_source_mass": 0.1111111119389534, "mixer_index": null, "normalized_entropy": 1.0000000180537645, "offset": null, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_8", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 } ], "mode": "detached_learned", "optimizer_state_after": "b82a349700eda58e2330a22ca2ee1f7fb1406efd47fd9aa306122b368a114192", "optimizer_state_before": "b82a349700eda58e2330a22ca2ee1f7fb1406efd47fd9aa306122b368a114192", "optimizer_state_unchanged": true, "positions": { "attention_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "676759a2d7c0615ad7e1f246e90927fb829d00c8b6c3745032ae6b27ab7d902a", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.0022265110164880753, 0.0021613112185150385, 0.0025601957459002733, 0.0025323182344436646, 0.0027253651060163975, 0.002904455177485943, 0.0029208383057266474, 0.00299624796025455, 0.0030937574338167906, 0.0033804215490818024, 0.0034370317589491606, 0.0038969130255281925, 0.004255065228790045, 0.004119740333408117, 0.003970506135374308, 0.004385598469525576, 0.0041999719105660915, 0.005232699681073427, 0.004664790350943804, 0.004898969549685717, 0.005131533369421959, 0.005069608334451914, 0.0051318020559847355, 0.0058372304774820805, 0.0059690922498703, 0.00554921617731452, 0.0058550527319312096, 0.006255472078919411, 0.006502739619463682, 0.006122888531535864, 0.007071305997669697, 0.006241494789719582 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.00015962477027642308, "first_to_last_ratio": 3.7699040072151497, "imbalance_abs_log_ratio": 1.3270495388586014, "last_quartile_mean": 4.234186599205714e-05, "mean": 8.617327489446325e-05, "normalized": [ 2.5428430303860874, 2.3222064178412967, 2.1011513502014654, 1.8911470095267005, 1.7004447966680725, 1.5353700031893112, 1.4229335846259188, 1.3028620492588128, 1.2049362956920768, 1.1705197150696323, 1.0773612761873523, 1.0133200843661747, 0.9906112963967714, 0.9272530070793641, 0.8550207003106389, 0.8185600560490035, 0.7703816559177782, 0.7173918665339755, 0.6760157258502337, 0.6514535263006792, 0.6184138115448249, 0.6053606047045522, 0.5838821473278404, 0.5697017460729616, 0.5457761369570343, 0.5276086337508958, 0.5104655265174319, 0.4951752820428687, 0.47527333481689216, 0.465789239220051, 0.45896414036514044, 0.45180594922816086 ], "peak_layer": 1, "peak_normalized": 2.5428430303860874, "population_cv": 0.5705341151299013, "rest_mean": 9.280180475562466e-05, "spike_contrast": 0.5428688997682254, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.0379213644191625e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00021912511147093028, 0.00020011213200632483, 0.00018106309289578348, 0.00016296633111778647, 0.00014653289690613747, 0.00013230786134954542, 0.0001226188469445333, 0.00011227188952034339, 0.0001038333066389896, 0.00010086751717608422, 9.283974941354245e-05, 8.732111018616706e-05, 8.536421955795959e-05, 7.990442827576771e-05, 7.367993384832516e-05, 7.053800072753802e-05, 6.63863102090545e-05, 6.182000652188435e-05, 5.825448897667229e-05, 5.6137883802875876e-05, 5.329074338078499e-05, 5.216590579948388e-05, 5.031503678765148e-05, 4.909306517220102e-05, 4.703131708083674e-05, 4.5465763832908124e-05, 4.3988486140733585e-05, 4.2670875700423494e-05, 4.095585973118432e-05, 4.013858415419236e-05, 3.955044303438626e-05, 3.893359826179221e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.00021442661636683624, "first_to_last_ratio": 1.843144421770586, "imbalance_abs_log_ratio": 0.6114730379455638, "last_quartile_mean": 0.0001163373926829081, "mean": 0.00015221274247778638, "normalized": [ 1.7741446897870732, 1.648461275666186, 1.5301464224684427, 1.4217121808823485, 1.3265065496562456, 1.2454420544341354, 1.1897998831173078, 1.1336248235930768, 1.0869089599614004, 1.0643003125599004, 1.0227030055526172, 0.9941765648180964, 0.9788958472869692, 0.9511102301475627, 0.9228060651723192, 0.9052418824754537, 0.8846484396889256, 0.865292475366091, 0.8497311645646503, 0.8388691935254817, 0.8256303537004397, 0.8176325372729673, 0.8076852907347736, 0.8000670639426607, 0.7892588692968658, 0.7814415503461742, 0.7731202151361662, 0.7664994567889658, 0.758490072461763, 0.7529625284402992, 0.7485174912671357, 0.7441725498875049 ], "peak_layer": 1, "peak_normalized": 1.7741446897870732, "population_cv": 0.27364872432853743, "rest_mean": 0.00015762320577373935, "spike_contrast": 0.7803181014868814, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012299624067964033, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00027004742878489196, 0.0002509168116375804, 0.0002329077833564952, 0.00021640271006617695, 0.00020191119983792305, 0.00018957215070258826, 0.0001811027032090351, 0.00017255214333999902, 0.00016544139361940324, 0.00016200006939470768, 0.00015566842921543866, 0.0001513263414381072, 0.00014900042151566595, 0.0001447710965294391, 0.00014046284195501357, 0.0001377893495373428, 0.00013465476513374597, 0.00013170854072086513, 0.00012933991092722863, 0.0001276865805266425, 0.00012567146040964872, 0.00012445409083738923, 0.00012293999316170812, 0.00012178040196886286, 0.00012013525702059269, 0.00011894536146428436, 0.00011767874821089208, 0.00011667098442558199, 0.00011545185407157987, 0.00011461049143690616, 0.00011393390013836324, 0.00011327254469506443 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.19015566632151604, "first_to_last_ratio": 1.8431444167877922, "imbalance_abs_log_ratio": 0.6114730352421437, "last_quartile_mean": 0.10316916275769472, "mean": 0.13498378219082952, "normalized": [ 1.7741447141792424, 1.6484611693507762, 1.5301464164964773, 1.421712099711509, 1.3265065258282553, 1.245442086656075, 1.1897999853143832, 1.1336247998774232, 1.0869090145314644, 1.0643003539468234, 1.0227030148964142, 0.9941765578539046, 0.978895844117059, 0.9511102293531465, 0.9228060470726628, 0.9052419251832852, 0.8846484748200404, 0.8652925213936268, 0.8497311905993837, 0.8388692019864461, 0.8256303029992552, 0.8176324960789627, 0.8076852727084025, 0.800067049482784, 0.789258877083647, 0.7814415614650243, 0.7731201949085273, 0.7664994759042474, 0.758490077799444, 0.7529625178867759, 0.7485174642151933, 0.7441725362993381 ], "peak_layer": 1, "peak_normalized": 1.7741447141792424, "population_cv": 0.2736487184871982, "rest_mean": 0.13978183573042904, "spike_contrast": 0.7803180757143806, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.10907429307699204, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.23948076367378235, 0.2225155234336853, 0.20654495060443878, 0.1919080764055252, 0.17905686795711517, 0.16811448335647583, 0.16060370206832886, 0.1530209630727768, 0.14671508967876434, 0.14366328716278076, 0.13804832100868225, 0.13419771194458008, 0.1321350634098053, 0.12838445603847504, 0.12456385046243668, 0.12219297885894775, 0.11941319704055786, 0.116800457239151, 0.11469992995262146, 0.11323373764753342, 0.11144670099020004, 0.11036712676286697, 0.10902441293001175, 0.10799607634544373, 0.10653714835643768, 0.10548193752765656, 0.10435868799686432, 0.1034649983048439, 0.10238385945558548, 0.10163772851228714, 0.10103771835565567, 0.10045122355222702 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 6.883365585963475e-05, "first_to_last_ratio": 2.2497546817252747, "imbalance_abs_log_ratio": 0.8108211799277664, "last_quartile_mean": 3.059607183786284e-05, "mean": 4.691841348858361e-05, "normalized": [ 1.7359303646705315, 1.6475859007944922, 1.5536416135247637, 1.4921074478374423, 1.436565604934822, 1.343834173349448, 1.2787439414688093, 1.2483315192759759, 1.191227587726315, 1.1603086071531024, 1.0995306735006558, 1.059977946939621, 1.0583984121521266, 1.0176934649371145, 0.9754909410255406, 0.9631621800279858, 0.9242301483750994, 0.8752327792179452, 0.8454160845678106, 0.8232909667825862, 0.7863453139125789, 0.77746685672799, 0.7497386589361914, 0.7388508716017261, 0.7093180450413769, 0.6908354487677191, 0.6685471901846501, 0.6592354484095886, 0.631440334976849, 0.6235901145414114, 0.6199506557967245, 0.6139807028410055 ], "peak_layer": 1, "peak_normalized": 1.7359303646705315, "population_cv": 0.3220412433027962, "rest_mean": 4.907019663530971e-05, "spike_contrast": 0.7193528234378936, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.529878449626267e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 8.144709863699973e-05, 7.730211655143648e-05, 7.289439963642508e-05, 7.000731420703232e-05, 6.740137905580923e-05, 6.305056740529835e-05, 5.999663699185476e-05, 5.856973439222202e-05, 5.5890508519951254e-05, 5.4439839004771784e-05, 5.1588234782684594e-05, 4.9732483603293076e-05, 4.965837433701381e-05, 4.774856279254891e-05, 4.576848732540384e-05, 4.519004141911864e-05, 4.3363412260077894e-05, 4.106453343410976e-05, 3.966558142565191e-05, 3.862750600092113e-05, 3.6894074582960457e-05, 3.647751145763323e-05, 3.517654840834439e-05, 3.4665710700210184e-05, 3.328007733216509e-05, 3.241290323785506e-05, 3.136717350571416e-05, 3.0930281354812905e-05, 2.9626178729813546e-05, 2.925785884144716e-05, 2.9087101211189292e-05, 2.8807000489905477e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 0.0001734125216898974, "first_to_last_ratio": 1.495517223041696, "imbalance_abs_log_ratio": 0.4024721155990803, "last_quartile_mean": 0.00011595488103921525, "mean": 0.00013990532806928968, "normalized": [ 1.3768889945483427, 1.3340940553978193, 1.2910049686488183, 1.2527203273458163, 1.2147536525239127, 1.1769854337196524, 1.14744939519443, 1.1220955988146075, 1.0944022472959734, 1.0773022658134643, 1.0508559212107247, 1.03206209473486, 1.0208171898483585, 1.000561882500425, 0.9805161605170362, 0.9663129296065233, 0.9487424991891081, 0.931931984738623, 0.9181974345490227, 0.9075104531838204, 0.8943640941785878, 0.8862664021756573, 0.8757830774574916, 0.86790401991425, 0.8562894022841879, 0.8478155487572495, 0.8387227683262297, 0.8314011142413483, 0.8224071457687988, 0.8161875050523302, 0.8112199678685613, 0.8064334645939688 ], "peak_layer": 1, "peak_normalized": 1.3768889945483427, "population_cv": 0.16325957486062354, "rest_mean": 0.0001431148236776978, "spike_contrast": 0.8564734849544942, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001225740517838858, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0001926341064972803, 0.00018664686649572104, 0.00018061847367789596, 0.00017526224837638438, 0.0001699505082797259, 0.00016466653323732316, 0.00016053428407758474, 0.00015698715287726372, 0.00015311270544771105, 0.00015072032692842185, 0.00014702034241054207, 0.00014439098595175892, 0.00014281776384450495, 0.00013998393842484802, 0.00013717943511437625, 0.00013519232743419707, 0.00013273413060232997, 0.00013038225006312132, 0.00012846071331296116, 0.00012696554767899215, 0.0001251263020094484, 0.00012399339175317436, 0.0001225267187692225, 0.00012142439663875848, 0.00011979944974882528, 0.00011861391249112785, 0.00011734178406186402, 0.00011631744564510882, 0.00011505914153531194, 0.00011418898066040128, 0.00011349399574100971, 0.00011282433843007311 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 0.00014675117381557357, "first_to_last_ratio": 1.2676837465168667, "imbalance_abs_log_ratio": 0.23719141364257146, "last_quartile_mean": 0.00011576323686313117, "mean": 0.00013042020850662084, "normalized": [ 1.169988069421413, 1.156175152925158, 1.1417665260225645, 1.1306726331060184, 1.1183621029407895, 1.1062694829504214, 1.0940115056230622, 1.0844989956600726, 1.0711591662817939, 1.0579765491085218, 1.0443900226505625, 1.035471661013805, 1.0260498632741444, 1.013992390086173, 1.0062479316494695, 0.9977001177294099, 0.9863163473759529, 0.9769959732660659, 0.9687585669786032, 0.9602628595878113, 0.9503845997255937, 0.9426104617668664, 0.9330243108845692, 0.9259760938667744, 0.914797625470175, 0.906289421438531, 0.8956357003805738, 0.888470885240542, 0.8806216558595509, 0.8756991509238615, 0.8712858841028995, 0.8681382926882502 ], "peak_layer": 1, "peak_normalized": 1.169988069421413, "population_cv": 0.09123006364857297, "rest_mean": 0.00013202972385678786, "spike_contrast": 0.9219804606102014, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012172882561571896, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00015259008796419948, 0.00015078860451467335, 0.00014890942838974297, 0.0001474625605624169, 0.00014585701865144074, 0.00014427989663090557, 0.00014268120867200196, 0.00014144058513920754, 0.00013970080181024969, 0.0001379815221298486, 0.00013620956451632082, 0.00013504642993211746, 0.0001338176371064037, 0.00013224509893916547, 0.00013123506505507976, 0.0001301202573813498, 0.00012863558367826045, 0.00012742001854348928, 0.00012634569429792464, 0.00012523768236860633, 0.00012394935765769333, 0.00012293545296415687, 0.00012168522516731173, 0.00012076599523425102, 0.00011930809705518186, 0.00011819845531135798, 0.00011680899478960782, 0.00011587455810513347, 0.00011485085997264832, 0.00011420886585256085, 0.00011363328667357564, 0.00011322277714498341 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00030002388848515693, "first_to_last_ratio": 2.2836575149168885, "imbalance_abs_log_ratio": 0.8257783308640597, "last_quartile_mean": 0.0001313786706305109, "mean": 0.0001947244400071213, "normalized": [ 1.8998833386658438, 1.7835645331795598, 1.6713073472446418, 1.5703972875779748, 1.4780250735507643, 1.3761669788219328, 1.3066447563403325, 1.2401013786965793, 1.1762106459027857, 1.150547637187856, 1.0768735950666006, 1.0264900078881174, 1.0140784159549234, 0.9666871251601796, 0.9096841069789114, 0.8754807117769676, 0.8421876125778792, 0.8073809902734768, 0.77911667677541, 0.7607651063686129, 0.7403822030704592, 0.7308855611321243, 0.7152791523187013, 0.7043381880055872, 0.6944157299089508, 0.686864629891504, 0.680599049276497, 0.6764772714226681, 0.6702412094778135, 0.6672258960044359, 0.6635863560708457, 0.6581114274310631 ], "peak_layer": 1, "peak_normalized": 1.8998833386658438, "population_cv": 0.3592853832280775, "rest_mean": 0.00020492727344390005, "spike_contrast": 0.6813594750078023, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013962913944851608, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0003699537192005664, 0.0003473036049399525, 0.0003254443872720003, 0.00030579473241232336, 0.00028780760476365685, 0.00026797334430739284, 0.0002544356684666127, 0.0002414780465187505, 0.0002290369593538344, 0.00022403974435292184, 0.00020969360775779933, 0.00019988269195891917, 0.0001974658516701311, 0.00018823760910890996, 0.00017713772831484675, 0.00017047749133780599, 0.00016399451124016196, 0.0001572168112033978, 0.00015171305858530104, 0.00014813955931458622, 0.00014417050988413393, 0.00014232128160074353, 0.00013928233238402754, 0.0001371518592350185, 0.00013521971413865685, 0.00013374933041632175, 0.00013252926873974502, 0.00013172665785532445, 0.00013051234418526292, 0.00012992518895771354, 0.00012921648158226162, 0.000128150379168801 ] } } }, "mlp_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "afe29f3ef4d2992353f0bb59fe7c5c2c5050fca671a53184825a8d63cd83a2bc", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.0027073738165199757, 0.002722575329244137, 0.0026850115973502398, 0.0026861224323511124, 0.00271838391199708, 0.0026645169127732515, 0.0026859452482312918, 0.0027359756641089916, 0.0026767742820084095, 0.0027451682835817337, 0.0026999011170119047, 0.002663193503394723, 0.002626648638397455, 0.0026531771291047335, 0.0026517712976783514, 0.002676903735846281, 0.002670086920261383, 0.0026355870068073273, 0.0026610493659973145, 0.0026388936676084995, 0.0026469926815479994, 0.00264862016774714, 0.002641494618728757, 0.002674198243767023, 0.002557571977376938, 0.00252733426168561, 0.0025468147359788418, 0.0026856842450797558, 0.002551089273765683, 0.0025778315030038357, 0.0026001501828432083, 0.0025662274565547705 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.00015858857113926206, "first_to_last_ratio": 3.757867690304466, "imbalance_abs_log_ratio": 1.3238516928952826, "last_quartile_mean": 4.220174423608114e-05, "mean": 8.573221236929385e-05, "normalized": [ 2.5461641896458365, 2.3032502265274117, 2.0924715218207024, 1.8886722264368014, 1.7031663195050915, 1.537434754996766, 1.4223955235407997, 1.3049501874408618, 1.2055297161045848, 1.1692542365736316, 1.0769571962730058, 1.0144505847411098, 0.9913601784485884, 0.9317001386730132, 0.857757082878185, 0.8155634736467603, 0.7682612945963089, 0.7217806418748633, 0.678338533281248, 0.6511820384364709, 0.6192882306865346, 0.6053793146733001, 0.5848721334223365, 0.5718144052517429, 0.5462281883927541, 0.527515083157495, 0.5112993585545178, 0.496920907994256, 0.47722358366421574, 0.46552107689700334, 0.4601234027970099, 0.45317424906679377 ], "peak_layer": 1, "peak_normalized": 2.5461641896458365, "population_cv": 0.5686091737198813, "rest_mean": 9.231269224848668e-05, "spike_contrast": 0.5437781067692293, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.0197621021652594e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00021828828903380781, 0.00019746273756027222, 0.00017939221288543195, 0.0001619200484128669, 0.00014601621660403907, 0.000131807682919316, 0.00012194511509733275, 0.00011187626660102978, 0.00010335272963857278, 0.00010024275252362713, 9.232992306351662e-05, 8.697109296917915e-05, 8.499150135321543e-05, 7.987671415321529e-05, 7.353741239057854e-05, 6.992006092332304e-05, 6.586474046343938e-05, 6.187985127326101e-05, 5.8155463193543255e-05, 5.582727681030519e-05, 5.309295011102222e-05, 5.1900507969548926e-05, 5.0142381951445714e-05, 4.902291402686387e-05, 4.682935104938224e-05, 4.522503513726406e-05, 4.3834825191879645e-05, 4.260212881490588e-05, 4.0913433622336015e-05, 3.991015182691626e-05, 3.944739728467539e-05, 3.885163096128963e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.00021242323236947414, "first_to_last_ratio": 1.8345651540958352, "imbalance_abs_log_ratio": 0.6068074801791611, "last_quartile_mean": 0.0001157894184871111, "mean": 0.00015111673860701558, "normalized": [ 1.771707915810251, 1.6366668487510905, 1.5242909365839221, 1.4186705636795862, 1.3264449439106374, 1.2453857912414694, 1.1893415085703798, 1.1330085309323608, 1.0858463851390703, 1.062861239127963, 1.023237806768965, 0.9942126959777651, 0.978436257513136, 0.9531027445378829, 0.9232414968561398, 0.904013237249231, 0.8853704564278231, 0.8680164030054869, 0.8515402788738432, 0.8395817783841374, 0.8267435189825344, 0.8185143642831039, 0.809172703595369, 0.8007918347797284, 0.7909896391756025, 0.7821052393452608, 0.7753388667989797, 0.7682453290925442, 0.7604995795745633, 0.7549901567959042, 0.7508572350862155, 0.7467737131490535 ], "peak_layer": 1, "peak_normalized": 1.771707915810251, "population_cv": 0.2714170004285523, "rest_mean": 0.00015645500981783562, "spike_contrast": 0.7816309251520459, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012229007406858728, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0002677347220014781, 0.00024732775636948645, 0.00023034587502479553, 0.00021438486874103546, 0.00020044803386554122, 0.0001881986390799284, 0.00017972940986510366, 0.00017121655400842428, 0.00016408956435043365, 0.00016061612404882908, 0.0001546283601783216, 0.00015024218009784818, 0.00014785809617023915, 0.0001440297783119604, 0.00013951724395155907, 0.000136611532070674, 0.0001337942958343774, 0.00013117180787958205, 0.0001286819897359237, 0.00012687486014328897, 0.00012493478425312787, 0.00012369122123345733, 0.00012227953993715346, 0.00012101305037504062, 0.00011953177454415709, 0.00011818919301731512, 0.00011716668086592108, 0.00011609472858253866, 0.00011492421617731452, 0.00011409165017539635, 0.00011346709652571008, 0.00011285000800853595 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.18837904930114746, "first_to_last_ratio": 1.8345651581276443, "imbalance_abs_log_ratio": 0.6068074823768531, "last_quartile_mean": 0.10268321540206671, "mean": 0.13401183625683188, "normalized": [ 1.7717078885231814, 1.6366668119471057, 1.524290957183596, 1.418670514273678, 1.326444920903592, 1.2453858718827995, 1.1893415500975306, 1.1330085716119305, 1.0858463365841577, 1.062861212873729, 1.0232377434953237, 0.9942127340740732, 0.9784362443707608, 0.9531027271661117, 0.9232414917376033, 0.904013241653558, 0.885370477646398, 0.8680164390463408, 0.8515402681847718, 0.8395816969955481, 0.8267435345782925, 0.8185144279720658, 0.8091727254708025, 0.8007918405918669, 0.7909896328038464, 0.7821052665924377, 0.7753388466443417, 0.7682453528580236, 0.7604996016741158, 0.7549901615676585, 0.7508572336421532, 0.7467736753526057 ], "peak_layer": 1, "peak_normalized": 1.7717078885231814, "population_cv": 0.27141699472708714, "rest_mean": 0.1387458680956452, "spike_contrast": 0.7816309473985974, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.10844806432724, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.23742982745170593, 0.2193327248096466, 0.2042730301618576, 0.19011864066123962, 0.1777593195438385, 0.16689644753932953, 0.15938584506511688, 0.151836559176445, 0.1455162614583969, 0.1424359828233719, 0.13712596893310547, 0.13323627412319183, 0.13112203776836395, 0.12772704660892487, 0.1237252876162529, 0.1211484745144844, 0.11865012347698212, 0.11632447689771652, 0.11411647498607635, 0.11251388490200043, 0.11079341918230057, 0.10969062149524689, 0.10843872278928757, 0.10731558501720428, 0.10600197315216064, 0.10481136292219162, 0.10390458256006241, 0.1029539704322815, 0.10191594809293747, 0.10117761790752411, 0.10062375664710999, 0.10007651150226593 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 6.873187476230669e-05, "first_to_last_ratio": 2.248757500145603, "imbalance_abs_log_ratio": 0.8103778415280068, "last_quartile_mean": 3.0564378221242805e-05, "mean": 4.685761763312257e-05, "normalized": [ 1.7400006587426948, 1.6405194725375354, 1.5551306973978087, 1.4944078341288562, 1.4327724019957897, 1.3414187773842097, 1.2804088050749736, 1.2499327782691996, 1.1927955171655855, 1.1608931065609767, 1.0981756972550076, 1.0577423904261343, 1.0542786038595535, 1.0218640087393278, 0.9775807389332813, 0.9570972402588515, 0.9209982861589985, 0.8815092249082567, 0.8478232134430647, 0.8204952153707635, 0.7871780668180614, 0.7759097745574741, 0.7506963524238334, 0.7421155343193898, 0.7072927325438048, 0.6891814936654451, 0.6695399899996352, 0.6616343983211914, 0.6345111343049281, 0.6212687927046295, 0.6200231913288072, 0.6148038704019305 ], "peak_layer": 1, "peak_normalized": 1.7400006587426948, "population_cv": 0.3217536161653951, "rest_mean": 4.900405669845727e-05, "spike_contrast": 0.719671983430414, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.526684668031521e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 8.153228554874659e-05, 7.687083416385576e-05, 7.286971958819777e-05, 7.002439087955281e-05, 6.71363013680093e-05, 6.285568815656006e-05, 5.999690620228648e-05, 5.8568872191244736e-05, 5.5891556257847697e-05, 5.439668530016206e-05, 5.145789691596292e-05, 4.9563288484932855e-05, 4.9400983698433265e-05, 4.7882112994557247e-05, 4.580710447044112e-05, 4.484729652176611e-05, 4.3155785533599555e-05, 4.130542220082134e-05, 3.972697595600039e-05, 3.844645107164979e-05, 3.688528886414133e-05, 3.635728353401646e-05, 3.517584264045581e-05, 3.477376594673842e-05, 3.314205241622403e-05, 3.229340290999971e-05, 3.1373048841487616e-05, 3.10026116494555e-05, 2.9731680115219206e-05, 2.9111175535945222e-05, 2.9052809622953646e-05, 2.8808244678657502e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 0.0001719075116852764, "first_to_last_ratio": 1.4894635014227988, "imbalance_abs_log_ratio": 0.39841598895878805, "last_quartile_mean": 0.00011541572621354135, "mean": 0.00013894635094402474, "normalized": [ 1.3738985779819257, 1.3296082759737424, 1.2883384004275333, 1.2499940655466777, 1.2130730176151365, 1.1755503979222366, 1.147409009170091, 1.11990595257071, 1.0929037170303153, 1.075598370036652, 1.0512762916941902, 1.0312594730951, 1.0194418974669357, 1.0018996501730997, 0.9803460168062212, 0.9648004557293595, 0.9493010807852901, 0.9344788929945884, 0.9198953859218769, 0.908001147597105, 0.8952913731815082, 0.886953676542665, 0.8771894463102521, 0.8683888934041172, 0.8579399877904575, 0.8482799643655919, 0.8408636339472488, 0.8330870307470335, 0.82434015220157, 0.8181187963300758, 0.8135579418994486, 0.809009026741244 ], "peak_layer": 1, "peak_normalized": 1.3738985779819257, "population_cv": 0.16161423472023242, "rest_mean": 0.0001421073115866252, "spike_contrast": 0.8576417505420824, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012187716347398237, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0001908981939777732, 0.0001847442181315273, 0.00017900991952046752, 0.00017368211410939693, 0.00016855206922627985, 0.00016333843814209104, 0.00015942829486448318, 0.00015560684551019222, 0.0001518549834145233, 0.00014945046859793365, 0.00014607100456487387, 0.0001432897406630218, 0.00014164773165248334, 0.00013921030040364712, 0.000136215501697734, 0.0001340555027127266, 0.0001319019211223349, 0.00012984243221580982, 0.0001278161071240902, 0.00012616344611160457, 0.00012439746933523566, 0.00012323897681199014, 0.00012188227265141904, 0.00012065946793882176, 0.00011920763063244522, 0.00011786540562752634, 0.00011683493357850239, 0.00011575440294109285, 0.0001145390560850501, 0.00011367462138878182, 0.00011304090730845928, 0.00011240885214647278 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 0.00014552221000485588, "first_to_last_ratio": 1.262697055715921, "imbalance_abs_log_ratio": 0.2332499537229619, "last_quartile_mean": 0.00011524712863320019, "mean": 0.00012953460168318998, "normalized": [ 1.1676213631634658, 1.1525055688027706, 1.1391580058081985, 1.1289307982409096, 1.1166731577821778, 1.1054001772932092, 1.094813144753513, 1.0822854389705017, 1.0679308603491235, 1.0564894822224282, 1.0456838360716472, 1.0337072702557388, 1.0234945545465333, 1.0158819477934218, 1.0051772952892828, 0.9951467950359763, 0.9866643397779161, 0.9787559418439278, 0.970448736962688, 0.9607908684400951, 0.9509939230158972, 0.9428061355388626, 0.934924980051271, 0.9261034265480621, 0.9170582603044295, 0.9058562786860701, 0.8981778407016658, 0.8907564927732574, 0.8831168120837269, 0.8778216107141017, 0.8745192645108896, 0.8703053916682415 ], "peak_layer": 1, "peak_normalized": 1.1676213631634658, "population_cv": 0.08967236486753827, "rest_mean": 0.00013110875065817877, "spike_contrast": 0.9231588022206524, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012103419721825049, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00015124736819416285, 0.00014928934979252517, 0.000147560378536582, 0.00014623560127802193, 0.00014464781270362437, 0.00014318757166620344, 0.00014181618462316692, 0.00014019341324456036, 0.0001383339986205101, 0.00013685194426216185, 0.00013545223919209093, 0.00013390085950959474, 0.00013257795944809914, 0.00013159186346456409, 0.00013020524056628346, 0.00012890594371128827, 0.00012780717224813998, 0.00012678276107180864, 0.0001257066905964166, 0.0001244556624442339, 0.00012318661902099848, 0.0001221260172314942, 0.00012110513489460573, 0.0001199624384753406, 0.00011879077646881342, 0.00011733973224181682, 0.00011634510883595794, 0.00011538378748809919, 0.0001143941844929941, 0.00011370827269274741, 0.00011328050459269434, 0.0001127346622524783 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00029741139769612346, "first_to_last_ratio": 2.278072415841521, "imbalance_abs_log_ratio": 0.8233296537390045, "last_quartile_mean": 0.00013055396993877366, "mean": 0.00019328556300024502, "normalized": [ 1.889398576482247, 1.780233894901915, 1.6772594716902607, 1.566224958959574, 1.4762232610465842, 1.375571144634086, 1.3083686481730985, 1.2364401527267181, 1.173148897934763, 1.1508864184295666, 1.080457040307468, 1.0318582947152708, 1.0081957004611959, 0.9698994715426978, 0.9066127339785177, 0.8754550048258108, 0.8446146862114313, 0.8115724449181009, 0.7806677558079268, 0.7613480996239201, 0.7403134026279838, 0.7294052762839086, 0.7160186976339193, 0.7062575707090604, 0.6942777320753531, 0.6866005528973866, 0.6829938476675228, 0.6753403086482458, 0.6715967314167978, 0.6686860556999406, 0.6640286432895989, 0.6600445236791291 ], "peak_layer": 1, "peak_normalized": 1.889398576482247, "population_cv": 0.3579154260008552, "rest_mean": 0.00020340604748559632, "spike_contrast": 0.681567477924495, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013863494677934796, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0003651934675872326, 0.0003440935106482357, 0.00032419004128314555, 0.0003027286729775369, 0.0002853326441254467, 0.00026587804313749075, 0.00025288877077400684, 0.00023898603103589267, 0.00022675274522043765, 0.00022244972933549434, 0.0002088367473334074, 0.00019944331143051386, 0.00019486967357806861, 0.00018746756541077048, 0.00017523515271022916, 0.00016921281348913908, 0.0001632518251426518, 0.00015686523693148047, 0.00015089180669747293, 0.00014715759607497603, 0.00014309189282357693, 0.00014098350948188454, 0.00013839607709087431, 0.0001365093921776861, 0.0001341938623227179, 0.00013270997442305088, 0.00013201285037212074, 0.00013053353177383542, 0.0001298099523410201, 0.00012924736074637622, 0.00012834715016651899, 0.00012757707736454904 ] } } }, "post_attention_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "31853ffb07b24cc59e774bb20656840b4358b3080b42516d380f2dc0f46404c8", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.0022265110164880753, 0.0040087271481752396, 0.005379301495850086, 0.006410556845366955, 0.0027253651060163975, 0.004838948138058186, 0.006175771821290255, 0.00742313964292407, 0.0030937574338167906, 0.005150026176124811, 0.007019025273621082, 0.008143599145114422, 0.004255065228790045, 0.0066224560141563416, 0.008129785768687725, 0.010099578648805618, 0.0041999719105660915, 0.00719039561226964, 0.009030314162373543, 0.010736005380749702, 0.005131533369421959, 0.00787449348717928, 0.009912566281855106, 0.012113703414797783, 0.0059690922498703, 0.00876442901790142, 0.010806876234710217, 0.012353328987956047, 0.006502739619463682, 0.009470963850617409, 0.012123330496251583, 0.01384514570236206 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.00015962236011546338, "first_to_last_ratio": 3.7698422272252903, "imbalance_abs_log_ratio": 1.3270331510418396, "last_quartile_mean": 4.2341920561739244e-05, "mean": 8.61727822893954e-05, "normalized": [ 2.542734798705971, 2.3221448837042376, 2.101145461274622, 1.8911858524935854, 1.7004204056817789, 1.5354142426015858, 1.4229001770142031, 1.3028733810328659, 1.2049412416934622, 1.1705258152816296, 1.0773908232649334, 1.0133220774376726, 0.990609360100461, 0.9272534949358001, 0.8550385909364457, 0.8185651575045715, 0.7703972051466339, 0.7173947009730917, 0.6760256695667757, 0.6514523109028573, 0.618419077603279, 0.6053620388084041, 0.5838930841859182, 0.5697043695032278, 0.5457802700901755, 0.5276149005446632, 0.5104701332743111, 0.49517988582787786, 0.47527562953524777, 0.46579059316028215, 0.45896435764104826, 0.45181000957238127 ], "peak_layer": 1, "peak_normalized": 2.542734798705971, "population_cv": 0.5705202801130907, "rest_mean": 9.280119640472414e-05, "spike_contrast": 0.542873885449774, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.03793460666202e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00021911453222855926, 0.00020010568550787866, 0.00018106155039276928, 0.00016296874673571438, 0.00014652995741926134, 0.00013231091725174338, 0.0001226152671733871, 0.00011227222421439365, 0.00010383313929196447, 0.00010086746624438092, 9.284176485380158e-05, 8.732078276807442e-05, 8.536356472177431e-05, 7.99040135461837e-05, 7.368105434579775e-05, 7.053803710732609e-05, 6.638727063545957e-05, 6.181989738252014e-05, 5.825501284562051e-05, 5.613745815935545e-05, 5.329089253791608e-05, 5.216573117650114e-05, 5.0315691623836756e-05, 4.9093010602518916e-05, 4.703140439232811e-05, 4.546604395727627e-05, 4.398863165988587e-05, 4.267102849553339e-05, 4.095582335139625e-05, 4.013847137684934e-05, 3.955023566959426e-05, 3.893372559105046e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.00021442477373057045, "first_to_last_ratio": 1.8431254562763404, "imbalance_abs_log_ratio": 0.6114627481432037, "last_quartile_mean": 0.00011633759004325839, "mean": 0.00015221248349917005, "normalized": [ 1.7740805953138137, 1.6484457246979003, 1.5301434809496548, 1.4217241600875643, 1.3264958046529274, 1.2454577490392524, 1.1897861330408546, 1.1336265611690741, 1.086914920172498, 1.064301358569379, 1.0227209980571423, 0.9941753882589759, 0.9788979908211966, 0.9511107967643203, 0.9228143274453928, 0.9052447611165039, 0.8846530997418666, 0.8652930871753776, 0.8497381552743479, 0.8388656494639406, 0.8256334792987466, 0.8176309647342147, 0.8076951735882674, 0.8000661307359944, 0.7892605467734257, 0.78144613040264, 0.7731219607566042, 0.7664988488799842, 0.7584935618388989, 0.7529639051546407, 0.7485195296370439, 0.7441790263875565 ], "peak_layer": 1, "peak_normalized": 1.7740805953138137, "population_cv": 0.2736411578809574, "rest_mean": 0.00015762286892384981, "spike_contrast": 0.7803207938393821, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012299640220589936, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00027003721334040165, 0.00025091401766985655, 0.00023290693934541196, 0.0002164041652576998, 0.000201909220777452, 0.0001895742170745507, 0.00018110030214302242, 0.00017255211423616856, 0.00016544201935175806, 0.00016199995297938585, 0.00015567090304102749, 0.00015132590488065034, 0.0001490004942752421, 0.00014477093645837158, 0.00014046386058907956, 0.00013778955326415598, 0.0001346552453469485, 0.00013170840975362808, 0.00012934075493831187, 0.00012768582382705063, 0.00012567172234412283, 0.00012445363972801715, 0.00012294128828216344, 0.00012178005272289738, 0.00012013530795229599, 0.00011894585622940212, 0.00011767881369451061, 0.00011667069338727742, 0.00011545218876563013, 0.00011461050598882139, 0.00011393401655368507, 0.00011327333777444437 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.190154030919075, "first_to_last_ratio": 1.84312545379361, "imbalance_abs_log_ratio": 0.6114627467961817, "last_quartile_mean": 0.10316933691501617, "mean": 0.13498355145566165, "normalized": [ 1.774080628234443, 1.6484456620184311, 1.5301434020504485, 1.4217242444591272, 1.326495766996536, 1.2454576834375484, 1.189786122599316, 1.133626627255238, 1.0869149569673349, 1.0643014004698326, 1.0227209907477024, 0.9941753870542451, 0.9788979589698585, 0.9511107512164827, 0.9228143032197887, 0.9052447972745712, 0.8846531331847195, 0.8652931173486506, 0.8497382179112226, 0.8388656682549475, 0.8256334805754935, 0.8176309131099365, 0.8076951535479161, 0.8000661540388709, 0.7892605573866218, 0.7814461538045975, 0.7731219580171443, 0.7664988542586116, 0.7584935821787124, 0.7529638601652359, 0.7485195164462104, 0.7441789968002049 ], "peak_layer": 1, "peak_normalized": 1.774080628234443, "population_cv": 0.27364115345546325, "rest_mean": 0.13978153549962574, "spike_contrast": 0.7803207857775153, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.10907443761825561, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.2394717037677765, 0.22251304984092712, 0.2065441906452179, 0.19190938770771027, 0.1790551096200943, 0.16811630129814148, 0.16060155630111694, 0.1530209481716156, 0.14671564102172852, 0.1436631828546524, 0.13805051147937775, 0.13419732451438904, 0.13213512301445007, 0.1283843070268631, 0.1245647519826889, 0.12219315767288208, 0.11941362172365189, 0.11680033802986145, 0.11470068246126175, 0.1132330670952797, 0.11144693940877914, 0.11036672443151474, 0.10902556031942368, 0.10799577087163925, 0.10653719305992126, 0.10548237711191177, 0.1043587476015091, 0.10346473753452301, 0.10238415747880936, 0.10163773596286774, 0.10103782266378403, 0.10045192390680313 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 6.883367768750759e-05, "first_to_last_ratio": 2.249759357551583, "imbalance_abs_log_ratio": 0.810823258297238, "last_quartile_mean": 3.059601795030176e-05, "mean": 4.691846504556452e-05, "normalized": [ 1.7359059710084281, 1.6475789727938472, 1.5536456441204052, 1.4921194549590373, 1.4365657321885223, 1.3438513058553838, 1.2787343947807137, 1.2483299149152907, 1.1912354282513808, 1.1603083401306369, 1.099547144002948, 1.059972440022979, 1.0583955432739494, 1.0176800955770247, 0.9754963047746087, 0.9631595708780456, 0.9242400656751482, 0.8752301116137658, 0.8454245377074081, 0.8232853322610043, 0.7863494898171388, 0.7774642190168813, 0.7497486129019063, 0.7388483538626245, 0.709315327139865, 0.6908343794807076, 0.6685469983108738, 0.6592365073812129, 0.6314341746588822, 0.6235898169926835, 0.6199474545603838, 0.6139783610863121 ], "peak_layer": 1, "peak_normalized": 1.7359059710084281, "population_cv": 0.3220406003070692, "rest_mean": 4.9070239684725595e-05, "spike_contrast": 0.7193541792517962, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.52988819940947e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 8.144604362314567e-05, 7.730187644483522e-05, 7.28946688468568e-05, 7.000795449130237e-05, 6.740145909134299e-05, 6.305144052021205e-05, 5.999625500408001e-05, 5.8569723478285596e-05, 5.589093780145049e-05, 5.4439886298496276e-05, 5.158906424185261e-05, 4.973227987647988e-05, 4.9658294301480055e-05, 4.774798799189739e-05, 4.576878927764483e-05, 4.51899686595425e-05, 4.3363925215089694e-05, 4.1064453398576006e-05, 3.9666021621087566e-05, 3.86272840842139e-05, 3.689431105158292e-05, 3.6477427784120664e-05, 3.5177054087398574e-05, 3.466563066467643e-05, 3.3279986382694915e-05, 3.2412888685939834e-05, 3.136719897156581e-05, 3.093036502832547e-05, 2.9625922252307646e-05, 2.9257877031341195e-05, 2.9086982976878062e-05, 2.8806922273361124e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 0.00017341253987979144, "first_to_last_ratio": 1.4955148931299893, "imbalance_abs_log_ratio": 0.4024705576674927, "last_quartile_mean": 0.00011595507385209203, "mean": 0.00013990553861731314, "normalized": [ 1.3768818258178377, 1.3340879911950096, 1.2910054180628048, 1.2527236427111486, 1.2147542166907819, 1.1769889670731195, 1.1474411155803916, 1.1220953663134672, 1.0944075691302455, 1.0773007485615413, 1.0508678613623372, 1.0320596054419082, 1.0208161736522385, 1.000559648637946, 0.9805197815181176, 0.9663128275349431, 0.948744191771126, 0.9319305822463463, 0.9182019814348666, 0.9075038868226692, 0.8943650364968407, 0.886262676120959, 0.8757907045344768, 0.8679002174812178, 0.856288373659771, 0.8478178092769747, 0.8387215581135022, 0.8313975747674917, 0.8224084044020752, 0.8161864847722404, 0.8112200992010112, 0.8064376596145924 ], "peak_layer": 1, "peak_normalized": 1.3768818258178377, "population_cv": 0.16325888721442564, "rest_mean": 0.0001431150389921472, "spike_contrast": 0.8564734877369171, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001225742365932092, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0001926333934534341, 0.00018664629897102714, 0.00018061880837194622, 0.0001752629759721458, 0.00016995084297377616, 0.0001646672753849998, 0.00016053336730692536, 0.00015698735660407692, 0.00015311368042603135, 0.00015072034148033708, 0.00014702223415952176, 0.00014439085498452187, 0.0001428178366040811, 0.00013998383656144142, 0.00013718014815822244, 0.00013519251660909504, 0.00013273456715978682, 0.00013038225006312132, 0.00012846154277212918, 0.00012696482008323073, 0.00012512662215158343, 0.0001239930570591241, 0.00012252797023393214, 0.000121424047392793, 0.00011979948612861335, 0.00011861440725624561, 0.00011734179133782163, 0.0001163171255029738, 0.00011505949078127742, 0.00011418900976423174, 0.00011349418491590768, 0.00011282509512966499 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 0.00014675183047074825, "first_to_last_ratio": 1.2676545313577883, "imbalance_abs_log_ratio": 0.23716836728277432, "last_quartile_mean": 0.00011576642282307148, "mean": 0.00013041823922321782, "normalized": [ 1.1700698937843759, 1.1560440994564782, 1.1417398043557874, 1.130463200949706, 1.118403537277814, 1.1065304334184665, 1.0939881912841023, 1.0846815122144253, 1.0709468270677047, 1.0579907390333845, 1.0442838370166418, 1.035550673160169, 1.025959021745254, 1.0140203095307772, 1.0062047700154308, 0.9976631870190885, 0.9861765922450596, 0.9770016877798172, 0.9687702939454803, 0.9602422119882037, 0.9504995944091904, 0.9425883202531964, 0.9330014667281973, 0.9259385264566372, 0.9147443798098084, 0.906202796813466, 0.8958065504175365, 0.8885655303249116, 0.8806194435723184, 0.8757905905296257, 0.8713783728577705, 0.8681336045391749 ], "peak_layer": 1, "peak_normalized": 1.1700698937843759, "population_cv": 0.09121429133712369, "rest_mean": 0.0001320278307604086, "spike_contrast": 0.921975648780332, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001217264449223876, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0001525984553154558, 0.0001507692359155044, 0.000148903694935143, 0.00014743302017450333, 0.00014586022007279098, 0.00014431175077334046, 0.00014267601363826543, 0.00014146225294098258, 0.000139670999487862, 0.00013798128929920495, 0.00013619365927297622, 0.00013505469541996717, 0.00013380376913119107, 0.00013224674330558628, 0.00013122745440341532, 0.00013011347618885338, 0.00012861541472375393, 0.00012741883983835578, 0.0001263453159481287, 0.0001252330985153094, 0.0001239624834852293, 0.00012293070903979242, 0.00012168040848337114, 0.00012075927224941552, 0.00011929935135412961, 0.00011818537313956767, 0.00011682951299007982, 0.00011588515189941972, 0.00011484883725643158, 0.00011421906674513593, 0.00011364363308530301, 0.00011322045611450449 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00030004180916876066, "first_to_last_ratio": 2.2837506327398005, "imbalance_abs_log_ratio": 0.8258191057718037, "last_quartile_mean": 0.00013138116082700435, "mean": 0.00019473525253488333, "normalized": [ 1.9000096511818072, 1.783700741635724, 1.6708066910504589, 1.570366884717585, 1.4783058799917985, 1.376185321596543, 1.3064973297630273, 1.2402700043336092, 1.1767953849206394, 1.1505341943958838, 1.0771542571420398, 1.026098162637436, 1.013898063759585, 0.9669473025647412, 0.9095398902231254, 0.8755205777387188, 0.842301214161826, 0.8073124727024382, 0.7791991818813049, 0.760574607765185, 0.7402055397447304, 0.7308143413325056, 0.7152667122155245, 0.7043714155678553, 0.6942467749765324, 0.6868749151283189, 0.680655564574882, 0.676363713518914, 0.6701061029149729, 0.6672276319278244, 0.6636177363814285, 0.6582317375530352 ], "peak_layer": 1, "peak_normalized": 1.9000096511818072, "population_cv": 0.35930746026536914, "rest_mean": 0.00020494150898971216, "spike_contrast": 0.6812747128050888, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013962146767880766, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0003699988592416048, 0.0003473494143690914, 0.0003253649629186839, 0.0003058057918678969, 0.0002878782688640058, 0.0002679917961359024, 0.0002544210874475539, 0.00024152429250534624, 0.00022916354646440595, 0.000224049566895701, 0.00020975990628357977, 0.00019981748482678086, 0.000197441695490852, 0.0001882987271528691, 0.00017711948021315038, 0.00017049472080543637, 0.00016402573965024203, 0.00015721219824627042, 0.00015173754945863038, 0.00014811068831477314, 0.00014414411270990968, 0.0001423153153154999, 0.0001392876438330859, 0.00013716594548895955, 0.00013519432104658335, 0.00013375876005738974, 0.00013254763325676322, 0.0001317118585575372, 0.00013049328117631376, 0.00012993274140171707, 0.00012922976748086512, 0.00012818092363886535 ] } } }, "post_mlp_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "7fd9c911e93175896ad618842c120e77b3f017c17b7eec9c2bfb01fc6dbc8ac2", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.0034885562490671873, 0.004831995815038681, 0.005952076520770788, 0.006925336085259914, 0.00385080324485898, 0.005542940925806761, 0.006729288958013058, 0.007927131839096546, 0.004081308841705322, 0.005811207927763462, 0.007534808944910765, 0.008581922389566898, 0.00499920267611742, 0.007102519739419222, 0.008542952127754688, 0.010447698645293713, 0.005041283555328846, 0.007636935915797949, 0.009387043304741383, 0.011020516976714134, 0.005807018838822842, 0.008367395959794521, 0.01024632342159748, 0.012378672137856483, 0.006453148555010557, 0.009127611294388771, 0.011037957854568958, 0.012611051090061665, 0.007010402157902718, 0.009794782847166061, 0.012445803731679916, 0.014087481424212456 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.00015858902861509705, "first_to_last_ratio": 3.757869257587208, "imbalance_abs_log_ratio": 1.3238521099622325, "last_quartile_mean": 4.220184837322449e-05, "mean": 8.57325238712292e-05, "normalized": [ 2.5462257183789148, 2.3032637538343064, 2.0924289533379263, 1.8886492391629932, 1.7031256747388024, 1.5374176267930122, 1.4224231993460965, 1.3049597038569594, 1.2055618292025094, 1.169245405308266, 1.0769812048488787, 1.0144334047898542, 0.9913512297333369, 0.9317090592952362, 0.8577499774884012, 0.81555804918997, 0.7682725064221458, 0.7217807351250844, 0.6783324192661732, 0.6511800118936032, 0.6192916242842709, 0.6053728292385717, 0.5848720876072855, 0.5718124973482859, 0.5462302773879167, 0.5275173250132701, 0.5112998770983732, 0.49691833866078555, 0.47721658788927634, 0.46551989467389665, 0.46012635628943105, 0.45317260249616614 ], "peak_layer": 1, "peak_normalized": 2.5462257183789148, "population_cv": 0.5686111305901927, "rest_mean": 9.231303704803361e-05, "spike_contrast": 0.5437775023084314, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.019775271648541e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00021829435718245804, 0.00019746461475733668, 0.0001793892151908949, 0.0001619186659809202, 0.00014601326256524771, 0.00013180669338908046, 0.00012194793089292943, 0.00011187748896190897, 0.00010335585830034688, 0.00010024235962191597, 9.233231685357168e-05, 8.69699360919185e-05, 8.499104296788573e-05, 7.987776916706935e-05, 7.353707042057067e-05, 6.991984992055222e-05, 6.586594099644572e-05, 6.188008410390466e-05, 5.8155150327365845e-05, 5.582730591413565e-05, 5.309343396220356e-05, 5.190014053368941e-05, 5.014256021240726e-05, 4.90229285787791e-05, 4.682970029534772e-05, 4.522539165918715e-05, 4.383502891869284e-05, 4.2602063331287354e-05, 4.0912982512963936e-05, 3.991019548266195e-05, 3.944779382436536e-05, 3.885163096128963e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.00021242368711682502, "first_to_last_ratio": 1.8345658968465297, "imbalance_abs_log_ratio": 0.6068078850437921, "last_quartile_mean": 0.0001157896194854402, "mean": 0.0001511171253696375, "normalized": [ 1.7717451736709768, 1.636674793183293, 1.5242755762025908, 1.4186515254985965, 1.3264266232459345, 1.2453738409548603, 1.1893512719369177, 1.1330135273991528, 1.085860457802012, 1.0628542818802846, 1.023252039669589, 0.9942070699734702, 0.9784308644775869, 0.9531083940389613, 0.9232390376562555, 0.9040072643238835, 0.8853775311228679, 0.8680181295617843, 0.8515367513400655, 0.8395795332993111, 0.8267439067367663, 0.8185133286645813, 0.8091730400239778, 0.800791325997512, 0.7909899739933552, 0.7821072339266707, 0.7753403490721149, 0.7682422073314642, 0.7604971517068638, 0.7549863467424895, 0.7508596466770359, 0.7467718018887739 ], "peak_layer": 1, "peak_normalized": 1.7717451736709768, "population_cv": 0.2714186609524326, "rest_mean": 0.00015645541295978345, "spike_contrast": 0.7816308178118705, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012229037238284946, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0002677410375326872, 0.0002473295899108052, 0.00023034414334688336, 0.00021438254043459892, 0.00020044577831868082, 0.00018819731485564262, 0.00017973134526982903, 0.000171217747265473, 0.0001640921109355986, 0.00016061548376455903, 0.00015463090676348656, 0.00015024171443656087, 0.0001478576596127823, 0.00014403100067283958, 0.00013951722939964384, 0.00013661097909789532, 0.00013379570737015456, 0.0001311724045081064, 0.0001286817860091105, 0.00012687484559137374, 0.0001249351626029238, 0.00012369138130452484, 0.00012227990373503417, 0.00012101328320568427, 0.00011953213106608018, 0.0001181897969217971, 0.0001171672047348693, 0.00011609455395955592, 0.00011492414341773838, 0.0001140913664130494, 0.00011346775136189535, 0.00011285000800853595 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.18837944976985455, "first_to_last_ratio": 1.8345658800697515, "imbalance_abs_log_ratio": 0.6068078758989703, "last_quartile_mean": 0.10268339328467846, "mean": 0.1340121782850474, "normalized": [ 1.7717451751532867, 1.6366747548098712, 1.5242756140248972, 1.4186514377480737, 1.3264266357207033, 1.2453739091726768, 1.1893513017871604, 1.1330134634090196, 1.0858604665419038, 1.0628542749024097, 1.0232520332438115, 0.9942070832344971, 0.9784308561813967, 0.9531084116998004, 0.9232390798282929, 0.9040072094634526, 0.8853775581660035, 0.8680181154203164, 0.8515367049614931, 0.8395794986042229, 0.8267439263823189, 0.8185133396785964, 0.809173050926764, 0.8007913534932437, 0.7909900046670136, 0.782107217826699, 0.7753403703776178, 0.7682422246066585, 0.7604971603471317, 0.7549863443945886, 0.7508596538022533, 0.7467717694238248 ], "peak_layer": 1, "peak_normalized": 1.7717451751532867, "population_cv": 0.27141865758333467, "rest_mean": 0.13874622461972413, "spike_contrast": 0.7816308398662988, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.10844832807779312, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.23743543028831482, 0.21933434903621674, 0.20427149534225464, 0.19011656939983368, 0.17775732278823853, 0.16689527034759521, 0.15938755869865417, 0.15183760225772858, 0.14551852643489838, 0.14243541657924652, 0.13712823390960693, 0.1332358568906784, 0.1311216503381729, 0.12772813439369202, 0.1237252801656723, 0.12114797532558441, 0.1186513751745224, 0.1163249984383583, 0.11411628872156143, 0.11251387745141983, 0.11079375445842743, 0.10969075560569763, 0.10843904316425323, 0.107315793633461, 0.10600229352712631, 0.104811891913414, 0.10390505194664001, 0.10295381397008896, 0.1019158810377121, 0.10117736458778381, 0.10062433779239655, 0.10007651150226593 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 6.873207303215167e-05, "first_to_last_ratio": 2.248757814140914, "imbalance_abs_log_ratio": 0.8103779811585758, "last_quartile_mean": 3.0564462122129044e-05, "mean": 4.6857780716891284e-05, "normalized": [ 1.7400119939228862, 1.6405409364325143, 1.555121247708426, 1.4944054279879166, 1.4327475398601994, 1.3414100714975594, 1.2804101716466747, 1.2499370459040815, 1.192804641985012, 1.1608858830017554, 1.098191983597808, 1.057744066133373, 1.0542717513613309, 1.021869458338179, 0.9775749297615711, 0.9570927445976433, 0.9210091333285576, 0.8815125232807631, 0.847820728512489, 0.8204970180441978, 0.7871838673851995, 0.7759041238100245, 0.7506938949785397, 0.7421070509174572, 0.7072987335050381, 0.6891811912847667, 0.6695407652827761, 0.6616333377893114, 0.6345027924930273, 0.6212629426015038, 0.6200302724102036, 0.614801730639213 ], "peak_layer": 1, "peak_normalized": 1.7400119939228862, "population_cv": 0.32175451551289735, "rest_mean": 4.9004235565748616e-05, "spike_contrast": 0.7196709452950106, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.5266924533061685e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 8.153310045599937e-05, 7.687210745643824e-05, 7.28695304132998e-05, 7.002452184678987e-05, 6.713537004543468e-05, 6.28554989816621e-05, 5.9997179050697014e-05, 5.8569276006892323e-05, 5.589217835222371e-05, 5.439653614303097e-05, 5.1458839152473956e-05, 4.9563539505470544e-05, 4.9400834541302174e-05, 4.788253500009887e-05, 4.58069916930981e-05, 4.4847241952084005e-05, 4.315644400776364e-05, 4.130572051508352e-05, 3.972699778387323e-05, 3.8446669350378215e-05, 3.688568904181011e-05, 3.635714529082179e-05, 3.5175849916413426e-05, 3.477348946034908e-05, 3.3142448955914006e-05, 3.2293501135427505e-05, 3.13731943606399e-05, 3.1002669857116416e-05, 2.9731392714893445e-05, 2.9111002731951885e-05, 2.905324254243169e-05, 2.8808244678657502e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 0.00017190765356644988, "first_to_last_ratio": 1.4894621368061003, "imbalance_abs_log_ratio": 0.3984150727783497, "last_quartile_mean": 0.00011541592721187044, "mean": 0.00013894661265112518, "normalized": [ 1.3739040544627474, 1.3296102750467884, 1.2883342981472405, 1.2499918159027226, 1.2130627732820631, 1.175544413473833, 1.1474083142393565, 1.1199112790671786, 1.092909618037032, 1.0755929927697745, 1.0512861461262244, 1.0312571117882559, 1.0194388253067554, 1.0019061415077835, 0.980344379776871, 0.9647961249933756, 0.9493075664595542, 0.9344817410273759, 0.9198929201794295, 0.9079992279088673, 0.8952920956888009, 0.8869530532606651, 0.8771902552778059, 0.8683889858357474, 0.8579407806493952, 0.8482823463732029, 0.8408658204630546, 0.8330843095877946, 0.8243382853592501, 0.818115422616924, 0.8135611224181157, 0.8090075029660142 ], "peak_layer": 1, "peak_normalized": 1.3739040544627474, "population_cv": 0.16161403917150835, "rest_mean": 0.00014210757001786045, "spike_contrast": 0.8576421569620596, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012187744287075474, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00019089931447524577, 0.00018474484386388212, 0.00017900968668982387, 0.00017368212866131216, 0.0001685509632807225, 0.00016333791427314281, 0.00015942849859129637, 0.00015560787869617343, 0.00015185608936008066, 0.00014945000293664634, 0.00014607264893129468, 0.0001432896824553609, 0.00014164757158141583, 0.0001392114645568654, 0.00013621553080156446, 0.0001340551534667611, 0.00013190307072363794, 0.00012984307250007987, 0.0001278160052606836, 0.00012616341700777411, 0.0001243978040292859, 0.00012323912233114243, 0.00012188261462142691, 0.00012065970804542303, 0.00011920796532649547, 0.00011786595860030502, 0.00011683545744745061, 0.00011575424287002534, 0.00011453901242930442, 0.00011367436673026532, 0.00011304156214464456, 0.00011240885214647278 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 0.00014552358334185556, "first_to_last_ratio": 1.262638843036328, "imbalance_abs_log_ratio": 0.2332038508028458, "last_quartile_mean": 0.00011525352965691127, "mean": 0.00012953498708156985, "normalized": [ 1.1676304712442283, 1.1525836984023852, 1.1391521450612143, 1.1288261090295841, 1.1167600441451297, 1.1054423860149858, 1.094731137320412, 1.082319740349084, 1.067957453000188, 1.0563935463552665, 1.04560972624621, 1.0337959762176818, 1.0235535208863231, 1.0159040893770892, 1.0050628637064216, 0.9950792389307415, 0.9865814183778941, 0.9786230528286792, 0.9704418054120744, 0.9608381133357455, 0.9508407831226554, 0.9429749854457838, 0.9346839821902475, 0.9262276149685831, 0.9170542960948607, 0.9059488475652709, 0.8984648923618161, 0.8908037775288885, 0.8831483920181569, 0.8778579808382188, 0.8744051093280881, 0.8703028022960919 ], "peak_layer": 1, "peak_normalized": 1.1676304712442283, "population_cv": 0.0896642712448568, "rest_mean": 0.00013110964479252557, "spike_contrast": 0.9231344927669946, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012103183544240892, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00015124899800866842, 0.00014929991448298097, 0.00014756005839444697, 0.00014622247545048594, 0.00014465949789155275, 0.00014319346519187093, 0.00014180598373059183, 0.00014019827358424664, 0.00013833785487804562, 0.00013683992438018322, 0.0001354430423816666, 0.00013391274842433631, 0.0001325859921053052, 0.00013159512309357524, 0.00013019080506637692, 0.00012889757636003196, 0.00012779681128449738, 0.00012676592450588942, 0.00012570616672746837, 0.00012446215259842575, 0.00012316714855842292, 0.00012214825255796313, 0.00012107427755836397, 0.00011997888213954866, 0.00011879061639774591, 0.00011735207226593047, 0.0001163826382253319, 0.0001153902558144182, 0.00011439861555118114, 0.00011371332220733166, 0.00011326605454087257, 0.0001127346622524783 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00029742828155576717, "first_to_last_ratio": 2.2781890121807993, "imbalance_abs_log_ratio": 0.8233808344454435, "last_quartile_mean": 0.00013055469935352448, "mean": 0.00019329176166138495, "normalized": [ 1.8891850069388627, 1.7801603926271052, 1.6773544463608039, 1.566500262900107, 1.476456430986117, 1.3754925511314788, 1.3084504582544147, 1.236424592515421, 1.172965600373877, 1.150871569148191, 1.0802294364746856, 1.0319209662753672, 1.0080063247818516, 0.9698985570693867, 0.9067931019504507, 0.8756163462587222, 0.8444435806568995, 0.8113939671049883, 0.7809677247391495, 0.7613442367276676, 0.7402582678385915, 0.7293562882295931, 0.7162426695419845, 0.7062419232587185, 0.6940340549797547, 0.6866100033591334, 0.6831688143117768, 0.6753320518823956, 0.6715919825156708, 0.6686410475337363, 0.6640239865195754, 0.6600233567535221 ], "peak_layer": 1, "peak_normalized": 1.8891850069388627, "population_cv": 0.357915029863326, "rest_mean": 0.00020341356920888993, "spike_contrast": 0.6815376252628051, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001386340009048581, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00036516389809548855, 0.0003440903383307159, 0.0003242187958676368, 0.0003027915954589844, 0.00028538686456158757, 0.00026587137836031616, 0.0002529126941226423, 0.00023899068764876574, 0.00022672458726447076, 0.00022245399304665625, 0.0002087994507746771, 0.00019946182146668434, 0.00019483931828290224, 0.00018747340072877705, 0.00017527563613839447, 0.00016924942610785365, 0.00016322398732881993, 0.00015683576930314302, 0.0001509546273155138, 0.00014716156874783337, 0.0001430858246749267, 0.0001409785618307069, 0.00013844380737282336, 0.00013651074550580233, 0.00013415106514003128, 0.00013271605712361634, 0.00013205090363044292, 0.00013053612201474607, 0.00012981319741811603, 0.00012924280599690974, 0.00012835036613978446, 0.00012757707736454904 ] } } }, "pre_attention_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "de71d4cf2d509a1063f6eac7f50c4885e9a058dee7e4b993ad40f5527f528d00", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.02802751213312149, 0.01413465291261673, 0.014225807972252369, 0.01433455292135477, 0.01440478302538395, 0.009703479707241058, 0.00977881159633398, 0.009858808480203152, 0.009950309991836548, 0.00752287870272994, 0.007571155205368996, 0.007693085819482803, 0.007788818795233965, 0.006353048607707024, 0.006389216054230928, 0.0064707002602517605, 0.00656962301582098, 0.005553483963012695, 0.0056249531917274, 0.005688921548426151, 0.005768724251538515, 0.004966245498508215, 0.005057469941675663, 0.005098348017781973, 0.005216081161051989, 0.0046758754178881645, 0.004727107938379049, 0.004777040798217058, 0.004837613552808762, 0.0044186655431985855, 0.004510571248829365, 0.0045849340967834 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.0001580166108396952, "first_to_last_ratio": 1.9713740353643698, "imbalance_abs_log_ratio": 0.6787307795042399, "last_quartile_mean": 8.015557068574708e-05, "mean": 0.00011843875881822896, "normalized": [ 0.8918679597819287, 1.5814819593588525, 1.5722239987443252, 1.3246067564285042, 1.1252650527356378, 1.512743589128775, 1.3788112428745087, 1.2863035124436846, 1.097095055843533, 1.468154353173571, 1.1523254606068964, 1.0839888571909224, 1.054518217028451, 1.146335080549274, 1.1665208527784754, 0.8931136212023528, 0.8203804970547371, 1.073741223277434, 0.919654742394771, 0.8223971330190447, 0.7711288002244273, 0.8470775332584999, 0.8298711629738247, 0.7662487463744663, 0.7210382462310316, 0.7071376059209309, 0.7189559392079661, 0.727980518763353, 0.6111916361855597, 0.6986829946379066, 0.6244678178312321, 0.6046898327751223 ], "peak_layer": 2, "peak_normalized": 1.5814819593588525, "population_cv": 0.2904715201464359, "rest_mean": 0.00012310891098523926, "spike_contrast": 0.7572151875143335, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.321993711637333e-05, "top_five_layers": [ 2, 3, 6, 10, 7 ], "top_five_spike_overlap": 0 }, "values": [ 0.00010563173418631777, 0.0001873087603598833, 0.00018621225899551064, 0.00015688478015363216, 0.0001332749961875379, 0.00017916747310664505, 0.00016330469225067645, 0.00015234819147735834, 0.00012993857671972364, 0.00017388637934345752, 0.0001364799973089248, 0.0001283862948184833, 0.00012489582877606153, 0.00013577050413005054, 0.00013816128193866462, 0.00010577926877886057, 9.71648478298448e-05, 0.00012717257777694613, 0.00010892276623053476, 9.74036956904456e-05, 9.133153798757121e-05, 0.0001003268116619438, 9.828891052166e-05, 9.07535504666157e-05, 8.539887494407594e-05, 8.375250035896897e-05, 8.515224908478558e-05, 8.622110908618197e-05, 7.238877878990024e-05, 8.275114669231698e-05, 7.396119326585904e-05, 7.161871326388791e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.0001664285200604354, "first_to_last_ratio": 1.9719337364651075, "imbalance_abs_log_ratio": 0.6790146534176558, "last_quartile_mean": 8.439863722742302e-05, "mean": 0.0001245867656507471, "normalized": [ 0.8984334461398115, 1.5927952483854981, 1.5658595485525297, 1.3198735865176374, 1.131213894321221, 1.5125315092450449, 1.3796810696180641, 1.286366052942558, 1.0960606282761096, 1.4536884557404346, 1.1543342746982923, 1.0797559630746076, 1.0497530570909623, 1.1457182987306291, 1.1551962694199982, 0.9023721663305979, 0.8238710779144349, 1.0632503993742801, 0.9211624832492573, 0.8274335806257342, 0.7739032412019898, 0.850990856837025, 0.8281317613098801, 0.7681943949939335, 0.7208251570035091, 0.7089546837555952, 0.7203162530736348, 0.7236174125342815, 0.6145740290254166, 0.6994991973458914, 0.6274381906330746, 0.6042038120380651 ], "peak_layer": 2, "peak_normalized": 1.5927952483854981, "population_cv": 0.2889665774838781, "rest_mean": 0.00012946851159169995, "spike_contrast": 0.7586812913967177, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.822533756960183e-05, "top_five_layers": [ 2, 3, 6, 10, 7 ], "top_five_spike_overlap": 0 }, "values": [ 0.00011193291720701382, 0.0001984412083402276, 0.0001950853766174987, 0.000164438781212084, 0.00014093428035266697, 0.00018844140868168324, 0.00017189000209327787, 0.00016026418597903103, 0.00013655464863404632, 0.00018111034296453, 0.00014381477376446128, 0.0001345233031315729, 0.00013078533811494708, 0.00014274133718572557, 0.00014392216689884663, 0.00011242362961638719, 0.00010264343291055411, 0.0001324669283349067, 0.00011476465442683548, 0.00010308727360097691, 9.641810174798593e-05, 0.00010602219845168293, 0.00010317425767425448, 9.570685506332666e-05, 8.980527491075918e-05, 8.832637104205787e-05, 8.974187221610919e-05, 9.015315299620852e-05, 7.656779052922502e-05, 8.714834257261828e-05, 7.817049481673166e-05, 7.527579873567447e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.14759047608822584, "first_to_last_ratio": 1.9719337947467823, "imbalance_abs_log_ratio": 0.6790146829732503, "last_quartile_mean": 0.07484555337578058, "mean": 0.1104847895912826, "normalized": [ 0.8984334974418968, 1.5927952138130326, 1.565859646322909, 1.3198735575362357, 1.1312139236939245, 1.5125315007911495, 1.3796810264894532, 1.286366010451999, 1.0960606168617568, 1.4536884922319049, 1.1543342013148132, 1.0797558953237962, 1.0497530983161607, 1.1457182562234747, 1.1551962948455465, 0.9023721940039697, 0.8238711061937815, 1.0632504861804335, 0.9211625151546727, 0.8274335809310764, 0.7739032642161865, 0.8509909070635955, 0.8281317391228538, 0.7681943896830757, 0.7208251689669645, 0.7089546589522648, 0.7203162343708721, 0.7236173971219663, 0.6145740294939214, 0.6994991437375198, 0.6274381319743458, 0.6042038211744466 ], "peak_layer": 2, "peak_normalized": 1.5927952138130326, "population_cv": 0.28896658018674404, "rest_mean": 0.11481397047086998, "spike_contrast": 0.7586813040631772, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.08710721284151077, "top_five_layers": [ 2, 3, 6, 10, 7 ], "top_five_spike_overlap": 0 }, "values": [ 0.09926323592662811, 0.1759796440601349, 0.1730036735534668, 0.14582595229148865, 0.12498193234205246, 0.16711172461509705, 0.15243376791477203, 0.14212387800216675, 0.12109802663326263, 0.16061046719551086, 0.1275363713502884, 0.1192966029047966, 0.11598175019025803, 0.12658444046974182, 0.12763161957263947, 0.09969840198755264, 0.0910252258181572, 0.11747300624847412, 0.10177444666624069, 0.09141882508993149, 0.08550453931093216, 0.09402155131101608, 0.09149596095085144, 0.08487379550933838, 0.0796402171254158, 0.0783287063241005, 0.07958398759365082, 0.07994871586561203, 0.06790108233690262, 0.07728401571512222, 0.069322369992733, 0.06675533205270767 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 5.093040158499207e-05, "first_to_last_ratio": 0.8860093039525244, "imbalance_abs_log_ratio": 0.12102782735710801, "last_quartile_mean": 5.748291960117058e-05, "mean": 5.7093861016710434e-05, "normalized": [ 0.4337136137178416, 0.9185569667519601, 0.9444095926066538, 0.8728787927927439, 0.7849140301431445, 1.1296292591692207, 1.0726593944235638, 0.9796135302182263, 0.9386642177229508, 1.3046353648934683, 1.0461832118506482, 1.0032917361864164, 0.9662905330123907, 1.0760080966276753, 1.1828739682986207, 0.9087955578115051, 0.9194941496879511, 1.198292373503509, 1.0946132296674007, 1.0673695485587213, 0.9244260207272295, 1.0525679458300632, 1.125653431968099, 0.9999504891362015, 0.9629027708994021, 1.0063234987710272, 0.9975933861195996, 1.0791525148711678, 0.9412468232151083, 1.085173475571988, 1.0031556955576197, 0.9789667796878816 ], "peak_layer": 10, "peak_normalized": 1.3046353648934683, "population_cv": 0.1441687540611157, "rest_mean": 5.695535414647397e-05, "spike_contrast": 1.0155638391297448, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.784179811598733e-05, "top_five_layers": [ 10, 18, 15, 6, 23 ], "top_five_spike_overlap": 1 }, "values": [ 2.4762384782661684e-05, 5.2443963795667514e-05, 5.3919990023132414e-05, 4.9836020480142906e-05, 4.481377254705876e-05, 6.449489592341706e-05, 6.124226638348773e-05, 5.592991874436848e-05, 5.3591964388033375e-05, 7.448667020071298e-05, 5.973063889541663e-05, 5.728179894504137e-05, 5.516925739357248e-05, 6.143345672171563e-05, 6.75348419463262e-05, 5.1886647270293906e-05, 5.249747118796222e-05, 6.84151382301934e-05, 6.249569560168311e-05, 6.094024865888059e-05, 5.277905074763112e-05, 6.009516800986603e-05, 6.426790059776977e-05, 5.7091034250333905e-05, 5.4975836974335834e-05, 5.74548939766828e-05, 5.695645813830197e-05, 6.161298369988799e-05, 5.373941530706361e-05, 6.19567435933277e-05, 5.727403186028823e-05, 5.58929932594765e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 5.801741053801379e-05, "first_to_last_ratio": 0.954195857595535, "imbalance_abs_log_ratio": 0.046886327142486896, "last_quartile_mean": 6.08024129178375e-05, "mean": 6.296396202287724e-05, "normalized": [ 0.4796364558374551, 0.9568353419130547, 0.9804638447804884, 0.8949545354119591, 0.8237377234766136, 1.1597846916175047, 1.085649359633547, 0.9904450076993521, 0.9595078978373608, 1.2941748206155441, 1.0680753760655664, 1.0231559828609242, 0.9904019047543832, 1.096490963646882, 1.1850351594596897, 0.9422215946487361, 0.9418631353047864, 1.1705024179981143, 1.0926353878857977, 1.0382275689133567, 0.9320833313178482, 1.0784583323883348, 1.0985080196909318, 0.9917906747078287, 0.9476388143731139, 0.9855066582268529, 0.978396290111904, 1.028407839812513, 0.8982055143685478, 1.0282804954549911, 0.9451124964564701, 0.9138123627295472 ], "peak_layer": 10, "peak_normalized": 1.2941748206155441, "population_cv": 0.13431233496292436, "rest_mean": 6.285090866084729e-05, "spike_contrast": 1.0115120295379672, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.357445017783903e-05, "top_five_layers": [ 10, 15, 18, 6, 23 ], "top_five_spike_overlap": 1 }, "values": [ 3.019981159013696e-05, 6.0246144130360335e-05, 6.173388828756288e-05, 5.634988337988034e-05, 5.186579073779285e-05, 7.302463927771896e-05, 6.835678505012766e-05, 6.236234185053036e-05, 6.041441884008236e-05, 8.148637425620109e-05, 6.725025741616264e-05, 6.442195444833487e-05, 6.235962791834027e-05, 6.903941539349034e-05, 7.461450877599418e-05, 5.9326004702597857e-05, 5.930343468207866e-05, 7.369946979451925e-05, 6.879665306769311e-05, 6.537092122016475e-05, 5.8687659475253895e-05, 6.790400948375463e-05, 6.916641723364592e-05, 6.244707037694752e-05, 5.966709431959316e-05, 6.205140380188823e-05, 6.160370685392991e-05, 6.475263216998428e-05, 5.655457789544016e-05, 6.474461406469345e-05, 5.950802733423188e-05, 5.753724690293893e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 1.7888660295284353e-05, "first_to_last_ratio": 0.41855462748958633, "imbalance_abs_log_ratio": 0.8709478659353102, "last_quartile_mean": 4.2739129185065394e-05, "mean": 3.1986895351110434e-05, "normalized": [ 0.2593955501951102, 0.4997829486650695, 0.48480994473028793, 0.4968760363401999, 0.5474125689219601, 0.7788406177150305, 0.7554473604850604, 0.6514322393717875, 0.7578135841604356, 0.9674810188795085, 0.9032772556875033, 0.9019220651946442, 0.8311356963419073, 0.9587970177975769, 1.1401394869948454, 0.8953886492354074, 1.0262272842074007, 1.1413916918814533, 1.174037958462901, 1.194927715786966, 1.0267140632187253, 1.3434898267078448, 1.3729067467361273, 1.2011929483563188, 1.1735959904166378, 1.3266745689353452, 1.2739952988364465, 1.4049535244931144, 1.2752889026856997, 1.4680979746684197, 1.4182292846928506, 1.3483241791974143 ], "peak_layer": 30, "peak_normalized": 1.4680979746684197, "population_cv": 0.3184955307507873, "rest_mean": 3.066251991657299e-05, "spike_contrast": 1.276428773763581, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.913852269761264e-05, "top_five_layers": [ 30, 31, 28, 23, 32 ], "top_five_spike_overlap": 1 }, "values": [ 8.297258318634704e-06, 1.5986504877218977e-05, 1.5507564967265353e-05, 1.589352177688852e-05, 1.7510028555989265e-05, 2.491269333404489e-05, 2.416441566310823e-05, 2.083729486912489e-05, 2.4240103812189773e-05, 3.094671410508454e-05, 2.889303505071439e-05, 2.8849686714238487e-05, 2.6585450541460887e-05, 3.066893987124786e-05, 3.6469522456172854e-05, 2.8640703021665104e-05, 3.282582474639639e-05, 3.6509576602838933e-05, 3.755382931558415e-05, 3.822202779701911e-05, 3.2841395295690745e-05, 4.2974068492185324e-05, 4.391502443468198e-05, 3.842243313556537e-05, 3.7539692129939795e-05, 4.243620060151443e-05, 4.0751154301688075e-05, 4.494010136113502e-05, 4.0792532672639936e-05, 4.695989628089592e-05, 4.5364751713350415e-05, 4.3128704419359565e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00022091567916504573, "first_to_last_ratio": 1.232986382535498, "imbalance_abs_log_ratio": 0.20943917994887448, "last_quartile_mean": 0.00017917122386279516, "mean": 0.0002198539991695725, "normalized": [ 0.4672441326905094, 1.0299890566632133, 1.08986441571188, 1.0084939820045402, 0.8814237539069011, 1.2693826047962031, 1.1886514905905403, 1.1035827467935222, 1.0473307057033197, 1.4274044616309502, 1.1442998336482333, 1.0911283607583675, 1.1094038041384326, 1.1946873974128753, 1.2511466100570947, 0.9923165311956169, 0.9239703707694655, 1.2473578196113966, 1.1181336030529359, 1.0160745414007064, 0.9397394354798955, 0.9905446518023159, 1.0121913654482901, 0.9359943959598221, 0.8839446941554127, 0.8510229499852975, 0.8648513533654478, 0.8786898836618169, 0.7455559493639876, 0.8399702482950041, 0.7392957942269532, 0.7163130557190521 ], "peak_layer": 10, "peak_normalized": 1.4274044616309502, "population_cv": 0.18952709251819305, "rest_mean": 0.0002217885959436427, "spike_contrast": 0.9441746799407325, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00020940717658959329, "top_five_layers": [ 10, 6, 15, 18, 14 ], "top_five_spike_overlap": 0 }, "values": [ 0.00010272549116052687, 0.00022644721320830286, 0.00023961105034686625, 0.00022172143508214504, 0.0001937845372594893, 0.0002790788421407342, 0.0002613297838252038, 0.00024262708029709756, 0.00023025984410196543, 0.000313820579322055, 0.00025157889467664063, 0.00023988893372006714, 0.00024390686303377151, 0.000262656802078709, 0.00027506958576850593, 0.0002181647578254342, 0.00020313858112785965, 0.00027423660503700376, 0.0002458261442370713, 0.00022338805138133466, 0.00020660547306761146, 0.00021777520305477083, 0.00022253431961871684, 0.00020578211115207523, 0.0001943387760547921, 0.00018710079893935472, 0.0001901410287246108, 0.00019318348495289683, 0.00016391345707233995, 0.00018467081827111542, 0.00016253713693004102, 0.00015748428995721042 ] } } }, "pre_mlp_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "2999a257265024350fd2665e2a8d452a03cd662193b4245d15290408ad29922d", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.014061377383768559, 0.014165290631353855, 0.014281988143920898, 0.014350161887705326, 0.00966146495193243, 0.009741888381540775, 0.009820535778999329, 0.00991273857653141, 0.0074927168898284435, 0.007546746172010899, 0.00766373984515667, 0.007762390188872814, 0.006325984839349985, 0.006375366821885109, 0.006445344537496567, 0.006555322092026472, 0.005531130358576775, 0.005611324217170477, 0.0056777093559503555, 0.00574630219489336, 0.004946626722812653, 0.0050392793491482735, 0.005084683652967215, 0.005203457549214363, 0.004667261149734259, 0.004717131145298481, 0.004768412560224533, 0.004826710093766451, 0.0044052014127373695, 0.004504534415900707, 0.004574235528707504, 0.004644096828997135 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 3.729895206561196e-05, "first_to_last_ratio": 1.2469721805352876, "imbalance_abs_log_ratio": 0.22071835733540474, "last_quartile_mean": 2.9911615229138988e-05, "mean": 3.340667609563752e-05, "normalized": [ 1.308752778077389, 1.2456464596610868, 1.0236956931411572, 0.9922285647386262, 1.26903181062375, 1.1170846373362797, 1.0287316533628064, 0.9469237624012169, 1.1300261751959062, 1.1132698789554223, 1.0126240729566007, 0.9379202574439188, 1.1130503370514597, 1.070713488608195, 0.9897642722064988, 0.922085688720865, 1.036831181462568, 0.9898105546019274, 0.9395130252451869, 0.9001967292772246, 0.9975952535130206, 0.9404117749145336, 0.9224862220158919, 0.8885787590271026, 0.9461670180110204, 0.8974372095121397, 0.8904691498559152, 0.8684437934704528, 0.9244147277585869, 0.9015689206268677, 0.8740901912628669, 0.8604359589635168 ], "peak_layer": 1, "peak_normalized": 1.308752778077389, "population_cv": 0.1164905978879517, "rest_mean": 3.378375206213807e-05, "spike_contrast": 0.9285666618330386, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.137046587653458e-05, "top_five_layers": [ 1, 5, 2, 9, 6 ], "top_five_spike_overlap": 0 }, "values": [ 4.3721080146497115e-05, 4.161290780757554e-05, 3.419827044126578e-05, 3.314705827506259e-05, 4.239413465256803e-05, 3.73180846509058e-05, 3.4366505133220926e-05, 3.1633575417799875e-05, 3.7750418414361775e-05, 3.719064625329338e-05, 3.3828404411906376e-05, 3.1332798243965954e-05, 3.718331208801828e-05, 3.5768978705164045e-05, 3.306473445263691e-05, 3.080381793552078e-05, 3.463708344497718e-05, 3.3066280593629926e-05, 3.138600732199848e-05, 3.007258055731654e-05, 3.332634150865488e-05, 3.1416031561093405e-05, 3.0817198421573266e-05, 2.9684462788281962e-05, 3.1608295103069395e-05, 2.9980394174344838e-05, 2.9747614462394267e-05, 2.9011820515734144e-05, 3.0881623388268054e-05, 3.01184209092753e-05, 2.9200447897892445e-05, 2.874430538213346e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 6.705871965095866e-05, "first_to_last_ratio": 0.7180297289018083, "imbalance_abs_log_ratio": 0.33124430563536084, "last_quartile_mean": 9.33926785364747e-05, "mean": 8.228966771639534e-05, "normalized": [ 0.8685826408392191, 0.8249006353131848, 0.7154905339803773, 0.6884730939760333, 0.9470487095509537, 0.8638069620260335, 0.8256155016985178, 0.7853664463613442, 0.9851686390305121, 0.9692511243547932, 0.9083804947955202, 0.8729855457896263, 1.0422548534804916, 1.0255492616133721, 0.9871528463964067, 0.9398223255303129, 1.0958375595009802, 1.0726325633807163, 1.0495259774381736, 1.0230838788771643, 1.1576441025653552, 1.1142532603054807, 1.1008567438213033, 1.056908774400763, 1.159063578749664, 1.1192434432436902, 1.1032241584740294, 1.1060435702772593, 1.1786235078649367, 1.1628365878275773, 1.141840825041133, 1.1085318534950752 ], "peak_layer": 29, "peak_normalized": 1.1786235078649367, "population_cv": 0.1341844256762087, "rest_mean": 8.049536754023629e-05, "spike_contrast": 1.1426606459269575, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.197888866765424e-05, "top_five_layers": [ 29, 30, 25, 21, 31 ], "top_five_spike_overlap": 2 }, "values": [ 7.14753768988885e-05, 6.788079917896539e-05, 5.887747829547152e-05, 5.66542221349664e-05, 7.793232362018898e-05, 7.108238787623122e-05, 6.793962529627606e-05, 6.462754390668124e-05, 8.106919995043427e-05, 7.975935295689851e-05, 7.475032907677814e-05, 7.183769048424438e-05, 8.576680556870997e-05, 8.4392107964959e-05, 8.123247971525416e-05, 7.733766688033938e-05, 9.017610864248127e-05, 8.826657722238451e-05, 8.636514394311234e-05, 8.418923243880272e-05, 9.526214853394777e-05, 9.169153054244816e-05, 9.0589135652408e-05, 8.697267185198143e-05, 9.537895675748587e-05, 9.210217103827745e-05, 9.078394941752777e-05, 9.101595787797123e-05, 9.698853682493791e-05, 9.56894364207983e-05, 9.396170207764953e-05, 9.122071787714958e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.05946834245696664, "first_to_last_ratio": 0.7180297222365345, "imbalance_abs_log_ratio": 0.33124431491808765, "last_quartile_mean": 0.08282156102359295, "mean": 0.07297530013602227, "normalized": [ 0.8685826122359729, 0.8249005917343738, 0.71549052911611, 0.6884731350635278, 0.9470486699630107, 0.8638070113577097, 0.8256154769796886, 0.7853664269052657, 0.9851686341386658, 0.9692511556795396, 0.9083805454214465, 0.8729855579949275, 1.0422548040385637, 1.0255492365750694, 0.9871528950273681, 0.9398223256645052, 1.0958375053304956, 1.07263252629438, 1.0495259690486791, 1.0230837911446407, 1.1576441435704576, 1.1142533045483842, 1.1008568124470575, 1.0569088284972943, 1.1590636022344982, 1.119243411839159, 1.1032241423789504, 1.1060435591240758, 1.1786235037173105, 1.1628366076958818, 1.141840809884862, 1.1085318743481278 ], "peak_layer": 29, "peak_normalized": 1.1786235037173105, "population_cv": 0.13418442668798974, "rest_mean": 0.07138409617322462, "spike_contrast": 1.1426607032635474, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.08156780153512955, "top_five_layers": [ 29, 30, 25, 21, 31 ], "top_five_spike_overlap": 2 }, "values": [ 0.06338507682085037, 0.0601973682641983, 0.05221313610672951, 0.050241533666849136, 0.0691111609339714, 0.06303657591342926, 0.060249537229537964, 0.05731235072016716, 0.07189297676086426, 0.07073139399290085, 0.06628934293985367, 0.06370638310909271, 0.07605885714292526, 0.0748397633433342, 0.07203777879476547, 0.06858381628990173, 0.07996907085180283, 0.07827568054199219, 0.07658947259187698, 0.07465984672307968, 0.0844794288277626, 0.08131296932697296, 0.0803353562951088, 0.07712823897600174, 0.08458301424980164, 0.08167712390422821, 0.08050811290740967, 0.08071386069059372, 0.0860104039311409, 0.08485835045576096, 0.08332617580890656, 0.08089544624090195 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 7.408341275549901e-06, "first_to_last_ratio": 0.49124437919492936, "imbalance_abs_log_ratio": 0.7108135577065074, "last_quartile_mean": 1.5080765479069669e-05, "mean": 1.1844782648040564e-05, "normalized": [ 0.5657990507718705, 0.5382233871697658, 0.49942145823009426, 0.48906984415931193, 0.8062921756393574, 0.7122953827323945, 0.7013508394120195, 0.6911626993599347, 0.8940675881372303, 0.868722127857376, 0.9735911331661544, 0.854855860776803, 0.989483587379368, 1.0059088486455374, 1.0303729006301217, 0.9941025538400318, 1.1620912657632405, 1.127012997009919, 1.113401270263767, 1.066097770938846, 1.1826034542812638, 1.1643547935214302, 1.2049778184223279, 1.1791491538213539, 1.2365172470752577, 1.219921447624247, 1.2337761202582145, 1.2288278254433027, 1.3039217543779833, 1.3631114810840155, 1.2856635723507386, 1.3138525898567213 ], "peak_layer": 30, "peak_normalized": 1.3631114810840155, "population_cv": 0.2549101540724733, "rest_mean": 1.1420299651271543e-05, "spike_contrast": 1.237882653019464, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.4136990830593277e-05, "top_five_layers": [ 30, 32, 29, 31, 25 ], "top_five_spike_overlap": 1 }, "values": [ 6.701766778860474e-06, 6.3751390371180605e-06, 5.915538622502936e-06, 5.7929260037781205e-06, 9.550355571263935e-06, 8.436983989668079e-06, 8.307348252856173e-06, 8.186671948351432e-06, 1.0590036254143342e-05, 1.0289824786013924e-05, 1.1531975360412616e-05, 1.0125581866304856e-05, 1.1720218026312068e-05, 1.1914771675947122e-05, 1.220454305439489e-05, 1.1774928680097219e-05, 1.3764718460151926e-05, 1.334922399109928e-05, 1.3187996046326589e-05, 1.2627696378331166e-05, 1.4007680874783546e-05, 1.379152945446549e-05, 1.4272700354922563e-05, 1.3966765436634887e-05, 1.4646278032159898e-05, 1.4449704394792207e-05, 1.4613809980801307e-05, 1.4555198504240252e-05, 1.5444669770658948e-05, 1.614575921848882e-05, 1.5228405572997872e-05, 1.5562298358418047e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 5.6029601637419546e-05, "first_to_last_ratio": 0.6070213107493427, "imbalance_abs_log_ratio": 0.49919138022039955, "last_quartile_mean": 9.230252817360451e-05, "mean": 7.840498903988191e-05, "normalized": [ 0.6598133090619259, 0.6419677920829977, 0.6005256373772865, 0.5913463573822231, 0.8562376513989928, 0.8062968559250673, 0.7923156566726156, 0.768439227921366, 0.9790122729268586, 0.9665730330981341, 0.9163710065350301, 0.8881686280668529, 1.0631190324760253, 1.0511030519715439, 1.0188080221730096, 0.9712812246823179, 1.1361272291219966, 1.1138561419734825, 1.0918134348286874, 1.0639217496772315, 1.203414696914147, 1.1592426077267617, 1.1442664103435203, 1.097952968622282, 1.2042281788778202, 1.1625445130141736, 1.1452986211445626, 1.1473932954411665, 1.2233942816523278, 1.205318760675343, 1.1829469858778916, 1.146901364356358 ], "peak_layer": 29, "peak_normalized": 1.2233942816523278, "population_cv": 0.1898597249088972, "rest_mean": 7.605543874820728e-05, "spike_contrast": 1.1977126437531995, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.109256061492487e-05, "top_five_layers": [ 29, 30, 25, 21, 31 ], "top_five_spike_overlap": 2 }, "values": [ 5.173265526536852e-05, 5.033347770222463e-05, 4.708420601673424e-05, 4.636450466932729e-05, 6.713330367347226e-05, 6.321769615169615e-05, 6.212150037754327e-05, 6.024946924299002e-05, 7.675944652874023e-05, 7.578414806630462e-05, 7.184805872384459e-05, 6.963685154914856e-05, 8.335383608937263e-05, 8.241172326961532e-05, 7.987963181221858e-05, 7.615329377586022e-05, 8.907804294722155e-05, 8.733187860343605e-05, 8.560362039133906e-05, 8.341677312273532e-05, 9.435371612198651e-05, 9.089040395338088e-05, 8.971619536168873e-05, 8.608499047113582e-05, 9.441749716643244e-05, 9.114928980125114e-05, 8.97971258382313e-05, 8.996135875349864e-05, 9.592021524440497e-05, 9.450300422031432e-05, 9.274894546251744e-05, 8.992278890218586e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 4.8511469685763586e-05, "first_to_last_ratio": 0.5271352332510719, "imbalance_abs_log_ratio": 0.6402981537748061, "last_quartile_mean": 9.202850924339145e-05, "mean": 7.532146378252946e-05, "normalized": [ 0.548104082267993, 0.5416830199005969, 0.528884702689083, 0.5247448675895723, 0.7777813723908718, 0.7487613756360456, 0.7496059866168319, 0.7329069161583202, 0.9480676391902623, 0.9399123881532756, 0.9000936195631334, 0.883441930780847, 1.0560589316963704, 1.0534552095740957, 1.032095781768693, 0.9835620691098637, 1.1601754560133901, 1.1467788545782247, 1.1233340534265408, 1.0967375255670042, 1.2396083970871608, 1.1951467072205837, 1.1828066060767042, 1.1317734013234224, 1.244698956870482, 1.2025862582796845, 1.1881970083126314, 1.1882256015362047, 1.2689460104605979, 1.2589162611011824, 1.2307315535498626, 1.1921774555104672 ], "peak_layer": 29, "peak_normalized": 1.2689460104605979, "population_cv": 0.23486334953575583, "rest_mean": 7.254842300123225e-05, "spike_contrast": 1.244629176846487, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.029588400153444e-05, "top_five_layers": [ 29, 30, 25, 21, 31 ], "top_five_spike_overlap": 2 }, "values": [ 4.1284001781605184e-05, 4.080035796505399e-05, 3.9836369978729635e-05, 3.952455153921619e-05, 5.858363147126511e-05, 5.639780283672735e-05, 5.6461420172126964e-05, 5.520362174138427e-05, 7.140984234865755e-05, 7.079557690303773e-05, 6.77963689668104e-05, 6.654213939327747e-05, 7.954390457598493e-05, 7.934778841445222e-05, 7.773896504659206e-05, 7.408333476632833e-05, 8.738611359149218e-05, 8.637706196168438e-05, 8.461116522084922e-05, 8.26078758109361e-05, 9.336911898571998e-05, 9.002019942272455e-05, 8.909072494134307e-05, 8.524682925781235e-05, 9.375254740007222e-05, 9.058055729838088e-05, 8.949673792812973e-05, 8.949889161158353e-05, 9.557887096889317e-05, 9.482341556577012e-05, 9.270050213672221e-05, 8.979655103757977e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 9.61669284151867e-05, "first_to_last_ratio": 0.8297833624837898, "imbalance_abs_log_ratio": 0.18659062131596024, "last_quartile_mean": 0.00011589401856326731, "mean": 0.00010716335873439675, "normalized": [ 0.9380742209455786, 0.8898820858744699, 0.7837417114644826, 0.7583102664636657, 1.0822070619439073, 0.9520749081713696, 0.9107982467806989, 0.8640017807748708, 1.0669538303069324, 1.0266950649381863, 0.9594029855154438, 0.8901506143407661, 1.0795289063552718, 1.0379165674077444, 0.9706971366471856, 0.914979822300863, 1.052939971267624, 1.0225707935964177, 0.9851238030379684, 0.9636652893153618, 1.0990044510799908, 1.0523904893769196, 1.0422234788441622, 1.0049019599938585, 1.0962247910521359, 1.0599378706264646, 1.05039930496957, 1.0583562345651827, 1.1199788667078585, 1.1100529546792268, 1.0850973923636869, 1.0717171382921347 ], "peak_layer": 29, "peak_normalized": 1.1199788667078585, "population_cv": 0.09156717959150794, "rest_mean": 0.00010599351123716958, "spike_contrast": 1.0706366257223145, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00011348053521942347, "top_five_layers": [ 29, 30, 21, 25, 31 ], "top_five_spike_overlap": 2 }, "values": [ 0.00010052718425868079, 9.536275319987908e-05, 8.398839418077841e-05, 8.12630751170218e-05, 0.00011597294360399246, 0.00010202754492638633, 9.760419925441965e-05, 9.25893327803351e-05, 0.00011433835607022047, 0.00011002409155480564, 0.00010281284630764276, 9.539152961224318e-05, 0.000115685943455901, 0.0001112266254494898, 0.0001040231654769741, 9.805231093196198e-05, 0.00011283658386673778, 0.00010958212078548968, 0.00010556917550275102, 0.00010326960909878835, 0.00011777300824178383, 0.0001127776995417662, 0.00011168816854478791, 0.00010768866923172027, 0.00011747513053705916, 0.00011358650226611644, 0.00011256431753281504, 0.00011341700883349404, 0.00012002069706795737, 0.00011895700299646705, 0.00011628268111962825, 0.00011484880815260112 ] } } } }, "selector": { "expected_uniform_count": 0, "identities_unique": true, "identity_order_sha256": "97fd9721a263024f6a746638f32e0ad9a988cd4c111c00e9e178c1022e5c9d5e", "passed": true, "uniform_count": 0, "uniform_indices": [], "visit_count": 65, "visits": [ { "branch": "attention", "coefficient": "detached_learned", "group": 1, "index": 0, "kind": "depth", "layer": 1, "offset": 1, "sources": 1, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 1, "index": 1, "kind": "depth", "layer": 1, "offset": 1, "sources": 2, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 1, "index": 2, "kind": "depth", "layer": 2, "offset": 2, "sources": 2, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 1, "index": 3, "kind": "depth", "layer": 2, "offset": 2, "sources": 2, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 1, "index": 4, "kind": "depth", "layer": 3, "offset": 3, "sources": 2, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 1, "index": 5, "kind": "depth", "layer": 3, "offset": 3, "sources": 2, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 1, "index": 6, "kind": "depth", "layer": 4, "offset": 4, "sources": 2, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 1, "index": 7, "kind": "depth", "layer": 4, "offset": 4, "sources": 2, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 2, "index": 8, "kind": "depth", "layer": 5, "offset": 1, "sources": 2, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 2, "index": 9, "kind": "depth", "layer": 5, "offset": 1, "sources": 3, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 2, "index": 10, "kind": "depth", "layer": 6, "offset": 2, "sources": 3, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 2, "index": 11, "kind": "depth", "layer": 6, "offset": 2, "sources": 3, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 2, "index": 12, "kind": "depth", "layer": 7, "offset": 3, "sources": 3, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 2, "index": 13, "kind": "depth", "layer": 7, "offset": 3, "sources": 3, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 2, "index": 14, "kind": "depth", "layer": 8, "offset": 4, "sources": 3, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 2, "index": 15, "kind": "depth", "layer": 8, "offset": 4, "sources": 3, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 3, "index": 16, "kind": "depth", "layer": 9, "offset": 1, "sources": 3, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 3, "index": 17, "kind": "depth", "layer": 9, "offset": 1, "sources": 4, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 3, "index": 18, "kind": "depth", "layer": 10, "offset": 2, "sources": 4, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 3, "index": 19, "kind": "depth", "layer": 10, "offset": 2, "sources": 4, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 3, "index": 20, "kind": "depth", "layer": 11, "offset": 3, "sources": 4, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 3, "index": 21, "kind": "depth", "layer": 11, "offset": 3, "sources": 4, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 3, "index": 22, "kind": "depth", "layer": 12, "offset": 4, "sources": 4, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 3, "index": 23, "kind": "depth", "layer": 12, "offset": 4, "sources": 4, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 4, "index": 24, "kind": "depth", "layer": 13, "offset": 1, "sources": 4, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 4, "index": 25, "kind": "depth", "layer": 13, "offset": 1, "sources": 5, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 4, "index": 26, "kind": "depth", "layer": 14, "offset": 2, "sources": 5, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 4, "index": 27, "kind": "depth", "layer": 14, "offset": 2, "sources": 5, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 4, "index": 28, "kind": "depth", "layer": 15, "offset": 3, "sources": 5, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 4, "index": 29, "kind": "depth", "layer": 15, "offset": 3, "sources": 5, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 4, "index": 30, "kind": "depth", "layer": 16, "offset": 4, "sources": 5, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 4, "index": 31, "kind": "depth", "layer": 16, "offset": 4, "sources": 5, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 5, "index": 32, "kind": "depth", "layer": 17, "offset": 1, "sources": 5, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 5, "index": 33, "kind": "depth", "layer": 17, "offset": 1, "sources": 6, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 5, "index": 34, "kind": "depth", "layer": 18, "offset": 2, "sources": 6, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 5, "index": 35, "kind": "depth", "layer": 18, "offset": 2, "sources": 6, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 5, "index": 36, "kind": "depth", "layer": 19, "offset": 3, "sources": 6, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 5, "index": 37, "kind": "depth", "layer": 19, "offset": 3, "sources": 6, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 5, "index": 38, "kind": "depth", "layer": 20, "offset": 4, "sources": 6, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 5, "index": 39, "kind": "depth", "layer": 20, "offset": 4, "sources": 6, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 6, "index": 40, "kind": "depth", "layer": 21, "offset": 1, "sources": 6, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 6, "index": 41, "kind": "depth", "layer": 21, "offset": 1, "sources": 7, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 6, "index": 42, "kind": "depth", "layer": 22, "offset": 2, "sources": 7, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 6, "index": 43, "kind": "depth", "layer": 22, "offset": 2, "sources": 7, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 6, "index": 44, "kind": "depth", "layer": 23, "offset": 3, "sources": 7, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 6, "index": 45, "kind": "depth", "layer": 23, "offset": 3, "sources": 7, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 6, "index": 46, "kind": "depth", "layer": 24, "offset": 4, "sources": 7, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 6, "index": 47, "kind": "depth", "layer": 24, "offset": 4, "sources": 7, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 7, "index": 48, "kind": "depth", "layer": 25, "offset": 1, "sources": 7, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 7, "index": 49, "kind": "depth", "layer": 25, "offset": 1, "sources": 8, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 7, "index": 50, "kind": "depth", "layer": 26, "offset": 2, "sources": 8, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 7, "index": 51, "kind": "depth", "layer": 26, "offset": 2, "sources": 8, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 7, "index": 52, "kind": "depth", "layer": 27, "offset": 3, "sources": 8, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 7, "index": 53, "kind": "depth", "layer": 27, "offset": 3, "sources": 8, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 7, "index": 54, "kind": "depth", "layer": 28, "offset": 4, "sources": 8, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 7, "index": 55, "kind": "depth", "layer": 28, "offset": 4, "sources": 8, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 8, "index": 56, "kind": "depth", "layer": 29, "offset": 1, "sources": 8, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 8, "index": 57, "kind": "depth", "layer": 29, "offset": 1, "sources": 9, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 8, "index": 58, "kind": "depth", "layer": 30, "offset": 2, "sources": 9, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 8, "index": 59, "kind": "depth", "layer": 30, "offset": 2, "sources": 9, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 8, "index": 60, "kind": "depth", "layer": 31, "offset": 3, "sources": 9, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 8, "index": 61, "kind": "depth", "layer": 31, "offset": 3, "sources": 9, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 8, "index": 62, "kind": "depth", "layer": 32, "offset": 4, "sources": 9, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 8, "index": 63, "kind": "depth", "layer": 32, "offset": 4, "sources": 9, "use_uniform": false }, { "branch": "output", "coefficient": "detached_learned", "group": null, "index": 64, "kind": "output", "layer": null, "offset": null, "sources": 9, "use_uniform": false } ], "visits_sha256": "5675dee7ec78bda7aa686371971870d4075869f1cf0f2f1b2e5e04bb1a176220" } }, "uniform_all": { "bits_per_byte": 7.959341947457508, "forward": { "activation_sha256": { "attention_branch_output": "676759a2d7c0615ad7e1f246e90927fb829d00c8b6c3745032ae6b27ab7d902a", "mlp_branch_output": "afe29f3ef4d2992353f0bb59fe7c5c2c5050fca671a53184825a8d63cd83a2bc", "post_attention_state": "31853ffb07b24cc59e774bb20656840b4358b3080b42516d380f2dc0f46404c8", "post_mlp_state": "7fd9c911e93175896ad618842c120e77b3f017c17b7eec9c2bfb01fc6dbc8ac2", "pre_attention_input": "de71d4cf2d509a1063f6eac7f50c4885e9a058dee7e4b993ad40f5527f528d00", "pre_mlp_input": "2999a257265024350fd2665e2a8d452a03cd662193b4245d15290408ad29922d" }, "logits_sha256": "873a309a197bfd08fa523edcb440d2697903a34abc02dad1c7f5991509c3c151", "loss_nats": 5.516995429992676, "mixer_summary_sha256": "8450b24bdc777885a3357dc7b8080e7410acbe96943686433da76f379295133a" }, "loss_nats": 5.516995429992676, "loss_scale": 1.0, "mixers": [ { "branch": "attention", "entropy_mean": -0.0, "group": 1, "latest_source_mass": 1.0, "layer": 1, "max_source_mass": 1.0, "mixer_index": 0, "normalized_entropy": 1.0, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 1.0, "median": 1.0, "p05": 1.0, "p95": 1.0 } ], "sources": 1 }, { "branch": "mlp", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 1, "max_source_mass": 0.5, "mixer_index": 1, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 2, "max_source_mass": 0.5, "mixer_index": 2, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 2, "max_source_mass": 0.5, "mixer_index": 3, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 3, "max_source_mass": 0.5, "mixer_index": 4, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 3, "max_source_mass": 0.5, "mixer_index": 5, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 4, "max_source_mass": 0.5, "mixer_index": 6, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 4, "max_source_mass": 0.5, "mixer_index": 7, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6931471824645996, "group": 2, "latest_source_mass": 0.5, "layer": 5, "max_source_mass": 0.5, "mixer_index": 8, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "completed_group_1", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 5, "max_source_mass": 0.3333333432674408, "mixer_index": 9, "normalized_entropy": 1.0000000180537645, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 6, "max_source_mass": 0.3333333432674408, "mixer_index": 10, "normalized_entropy": 1.0000000180537645, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 6, "max_source_mass": 0.3333333432674408, "mixer_index": 11, "normalized_entropy": 1.0000000180537645, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 7, "max_source_mass": 0.3333333432674408, "mixer_index": 12, "normalized_entropy": 1.0000000180537645, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 7, "max_source_mass": 0.3333333432674408, "mixer_index": 13, "normalized_entropy": 1.0000000180537645, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 8, "max_source_mass": 0.3333333432674408, "mixer_index": 14, "normalized_entropy": 1.0000000180537645, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 8, "max_source_mass": 0.3333333432674408, "mixer_index": 15, "normalized_entropy": 1.0000000180537645, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0986123085021973, "group": 3, "latest_source_mass": 0.3333333432674408, "layer": 9, "max_source_mass": 0.3333333432674408, "mixer_index": 16, "normalized_entropy": 1.0000000180537645, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_2", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 9, "max_source_mass": 0.25, "mixer_index": 17, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 10, "max_source_mass": 0.25, "mixer_index": 18, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 10, "max_source_mass": 0.25, "mixer_index": 19, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 11, "max_source_mass": 0.25, "mixer_index": 20, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 11, "max_source_mass": 0.25, "mixer_index": 21, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 12, "max_source_mass": 0.25, "mixer_index": 22, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 12, "max_source_mass": 0.25, "mixer_index": 23, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3862943649291992, "group": 4, "latest_source_mass": 0.25, "layer": 13, "max_source_mass": 0.25, "mixer_index": 24, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_3", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 13, "max_source_mass": 0.20000000298023224, "mixer_index": 25, "normalized_entropy": 1.0000000186840277, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 14, "max_source_mass": 0.20000000298023224, "mixer_index": 26, "normalized_entropy": 1.0000000186840277, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 14, "max_source_mass": 0.20000000298023224, "mixer_index": 27, "normalized_entropy": 1.0000000186840277, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 15, "max_source_mass": 0.20000000298023224, "mixer_index": 28, "normalized_entropy": 1.0000000186840277, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 15, "max_source_mass": 0.20000000298023224, "mixer_index": 29, "normalized_entropy": 1.0000000186840277, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 16, "max_source_mass": 0.20000000298023224, "mixer_index": 30, "normalized_entropy": 1.0000000186840277, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 16, "max_source_mass": 0.20000000298023224, "mixer_index": 31, "normalized_entropy": 1.0000000186840277, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.6094379425048828, "group": 5, "latest_source_mass": 0.20000000298023224, "layer": 17, "max_source_mass": 0.20000000298023224, "mixer_index": 32, "normalized_entropy": 1.0000000186840277, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_4", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 17, "max_source_mass": 0.1666666716337204, "mixer_index": 33, "normalized_entropy": 1.0000001451965654, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 18, "max_source_mass": 0.1666666716337204, "mixer_index": 34, "normalized_entropy": 1.0000001451965654, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 18, "max_source_mass": 0.1666666716337204, "mixer_index": 35, "normalized_entropy": 1.0000001451965654, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 19, "max_source_mass": 0.1666666716337204, "mixer_index": 36, "normalized_entropy": 1.0000001451965654, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 19, "max_source_mass": 0.1666666716337204, "mixer_index": 37, "normalized_entropy": 1.0000001451965654, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 20, "max_source_mass": 0.1666666716337204, "mixer_index": 38, "normalized_entropy": 1.0000001451965654, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 20, "max_source_mass": 0.1666666716337204, "mixer_index": 39, "normalized_entropy": 1.0000001451965654, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.791759729385376, "group": 6, "latest_source_mass": 0.1666666716337204, "layer": 21, "max_source_mass": 0.1666666716337204, "mixer_index": 40, "normalized_entropy": 1.0000001451965654, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_5", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 21, "max_source_mass": 0.1428571492433548, "mixer_index": 41, "normalized_entropy": 1.00000003408302, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 22, "max_source_mass": 0.1428571492433548, "mixer_index": 42, "normalized_entropy": 1.00000003408302, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 22, "max_source_mass": 0.1428571492433548, "mixer_index": 43, "normalized_entropy": 1.00000003408302, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 23, "max_source_mass": 0.1428571492433548, "mixer_index": 44, "normalized_entropy": 1.00000003408302, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 23, "max_source_mass": 0.1428571492433548, "mixer_index": 45, "normalized_entropy": 1.00000003408302, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 24, "max_source_mass": 0.1428571492433548, "mixer_index": 46, "normalized_entropy": 1.00000003408302, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 24, "max_source_mass": 0.1428571492433548, "mixer_index": 47, "normalized_entropy": 1.00000003408302, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9459102153778076, "group": 7, "latest_source_mass": 0.1428571492433548, "layer": 25, "max_source_mass": 0.1428571492433548, "mixer_index": 48, "normalized_entropy": 1.00000003408302, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_6", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 25, "max_source_mass": 0.125, "mixer_index": 49, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 26, "max_source_mass": 0.125, "mixer_index": 50, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 26, "max_source_mass": 0.125, "mixer_index": 51, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 27, "max_source_mass": 0.125, "mixer_index": 52, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 27, "max_source_mass": 0.125, "mixer_index": 53, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 28, "max_source_mass": 0.125, "mixer_index": 54, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 28, "max_source_mass": 0.125, "mixer_index": 55, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.079441547393799, "group": 8, "latest_source_mass": 0.125, "layer": 29, "max_source_mass": 0.125, "mixer_index": 56, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_7", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 29, "max_source_mass": 0.1111111119389534, "mixer_index": 57, "normalized_entropy": 1.0000000180537645, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 30, "max_source_mass": 0.1111111119389534, "mixer_index": 58, "normalized_entropy": 1.0000000180537645, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 30, "max_source_mass": 0.1111111119389534, "mixer_index": 59, "normalized_entropy": 1.0000000180537645, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 31, "max_source_mass": 0.1111111119389534, "mixer_index": 60, "normalized_entropy": 1.0000000180537645, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 31, "max_source_mass": 0.1111111119389534, "mixer_index": 61, "normalized_entropy": 1.0000000180537645, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 32, "max_source_mass": 0.1111111119389534, "mixer_index": 62, "normalized_entropy": 1.0000000180537645, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 32, "max_source_mass": 0.1111111119389534, "mixer_index": 63, "normalized_entropy": 1.0000000180537645, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "output", "entropy_mean": 2.1972246170043945, "group": null, "latest_source_mass": 0.1111111119389534, "layer": null, "max_source_mass": 0.1111111119389534, "mixer_index": null, "normalized_entropy": 1.0000000180537645, "offset": null, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_8", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 } ], "mode": "uniform_all", "optimizer_state_after": "b82a349700eda58e2330a22ca2ee1f7fb1406efd47fd9aa306122b368a114192", "optimizer_state_before": "b82a349700eda58e2330a22ca2ee1f7fb1406efd47fd9aa306122b368a114192", "optimizer_state_unchanged": true, "positions": { "attention_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "676759a2d7c0615ad7e1f246e90927fb829d00c8b6c3745032ae6b27ab7d902a", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.0022265110164880753, 0.0021613112185150385, 0.0025601957459002733, 0.0025323182344436646, 0.0027253651060163975, 0.002904455177485943, 0.0029208383057266474, 0.00299624796025455, 0.0030937574338167906, 0.0033804215490818024, 0.0034370317589491606, 0.0038969130255281925, 0.004255065228790045, 0.004119740333408117, 0.003970506135374308, 0.004385598469525576, 0.0041999719105660915, 0.005232699681073427, 0.004664790350943804, 0.004898969549685717, 0.005131533369421959, 0.005069608334451914, 0.0051318020559847355, 0.0058372304774820805, 0.0059690922498703, 0.00554921617731452, 0.0058550527319312096, 0.006255472078919411, 0.006502739619463682, 0.006122888531535864, 0.007071305997669697, 0.006241494789719582 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.00015962477027642308, "first_to_last_ratio": 3.7699040072151497, "imbalance_abs_log_ratio": 1.3270495388586014, "last_quartile_mean": 4.234186599205714e-05, "mean": 8.617327489446325e-05, "normalized": [ 2.5428430303860874, 2.3222064178412967, 2.1011513502014654, 1.8911470095267005, 1.7004447966680725, 1.5353700031893112, 1.4229335846259188, 1.3028620492588128, 1.2049362956920768, 1.1705197150696323, 1.0773612761873523, 1.0133200843661747, 0.9906112963967714, 0.9272530070793641, 0.8550207003106389, 0.8185600560490035, 0.7703816559177782, 0.7173918665339755, 0.6760157258502337, 0.6514535263006792, 0.6184138115448249, 0.6053606047045522, 0.5838821473278404, 0.5697017460729616, 0.5457761369570343, 0.5276086337508958, 0.5104655265174319, 0.4951752820428687, 0.47527333481689216, 0.465789239220051, 0.45896414036514044, 0.45180594922816086 ], "peak_layer": 1, "peak_normalized": 2.5428430303860874, "population_cv": 0.5705341151299013, "rest_mean": 9.280180475562466e-05, "spike_contrast": 0.5428688997682254, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.0379213644191625e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00021912511147093028, 0.00020011213200632483, 0.00018106309289578348, 0.00016296633111778647, 0.00014653289690613747, 0.00013230786134954542, 0.0001226188469445333, 0.00011227188952034339, 0.0001038333066389896, 0.00010086751717608422, 9.283974941354245e-05, 8.732111018616706e-05, 8.536421955795959e-05, 7.990442827576771e-05, 7.367993384832516e-05, 7.053800072753802e-05, 6.63863102090545e-05, 6.182000652188435e-05, 5.825448897667229e-05, 5.6137883802875876e-05, 5.329074338078499e-05, 5.216590579948388e-05, 5.031503678765148e-05, 4.909306517220102e-05, 4.703131708083674e-05, 4.5465763832908124e-05, 4.3988486140733585e-05, 4.2670875700423494e-05, 4.095585973118432e-05, 4.013858415419236e-05, 3.955044303438626e-05, 3.893359826179221e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.00021442661636683624, "first_to_last_ratio": 1.843144421770586, "imbalance_abs_log_ratio": 0.6114730379455638, "last_quartile_mean": 0.0001163373926829081, "mean": 0.00015221274247778638, "normalized": [ 1.7741446897870732, 1.648461275666186, 1.5301464224684427, 1.4217121808823485, 1.3265065496562456, 1.2454420544341354, 1.1897998831173078, 1.1336248235930768, 1.0869089599614004, 1.0643003125599004, 1.0227030055526172, 0.9941765648180964, 0.9788958472869692, 0.9511102301475627, 0.9228060651723192, 0.9052418824754537, 0.8846484396889256, 0.865292475366091, 0.8497311645646503, 0.8388691935254817, 0.8256303537004397, 0.8176325372729673, 0.8076852907347736, 0.8000670639426607, 0.7892588692968658, 0.7814415503461742, 0.7731202151361662, 0.7664994567889658, 0.758490072461763, 0.7529625284402992, 0.7485174912671357, 0.7441725498875049 ], "peak_layer": 1, "peak_normalized": 1.7741446897870732, "population_cv": 0.27364872432853743, "rest_mean": 0.00015762320577373935, "spike_contrast": 0.7803181014868814, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012299624067964033, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00027004742878489196, 0.0002509168116375804, 0.0002329077833564952, 0.00021640271006617695, 0.00020191119983792305, 0.00018957215070258826, 0.0001811027032090351, 0.00017255214333999902, 0.00016544139361940324, 0.00016200006939470768, 0.00015566842921543866, 0.0001513263414381072, 0.00014900042151566595, 0.0001447710965294391, 0.00014046284195501357, 0.0001377893495373428, 0.00013465476513374597, 0.00013170854072086513, 0.00012933991092722863, 0.0001276865805266425, 0.00012567146040964872, 0.00012445409083738923, 0.00012293999316170812, 0.00012178040196886286, 0.00012013525702059269, 0.00011894536146428436, 0.00011767874821089208, 0.00011667098442558199, 0.00011545185407157987, 0.00011461049143690616, 0.00011393390013836324, 0.00011327254469506443 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.19015566632151604, "first_to_last_ratio": 1.8431444167877922, "imbalance_abs_log_ratio": 0.6114730352421437, "last_quartile_mean": 0.10316916275769472, "mean": 0.13498378219082952, "normalized": [ 1.7741447141792424, 1.6484611693507762, 1.5301464164964773, 1.421712099711509, 1.3265065258282553, 1.245442086656075, 1.1897999853143832, 1.1336247998774232, 1.0869090145314644, 1.0643003539468234, 1.0227030148964142, 0.9941765578539046, 0.978895844117059, 0.9511102293531465, 0.9228060470726628, 0.9052419251832852, 0.8846484748200404, 0.8652925213936268, 0.8497311905993837, 0.8388692019864461, 0.8256303029992552, 0.8176324960789627, 0.8076852727084025, 0.800067049482784, 0.789258877083647, 0.7814415614650243, 0.7731201949085273, 0.7664994759042474, 0.758490077799444, 0.7529625178867759, 0.7485174642151933, 0.7441725362993381 ], "peak_layer": 1, "peak_normalized": 1.7741447141792424, "population_cv": 0.2736487184871982, "rest_mean": 0.13978183573042904, "spike_contrast": 0.7803180757143806, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.10907429307699204, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.23948076367378235, 0.2225155234336853, 0.20654495060443878, 0.1919080764055252, 0.17905686795711517, 0.16811448335647583, 0.16060370206832886, 0.1530209630727768, 0.14671508967876434, 0.14366328716278076, 0.13804832100868225, 0.13419771194458008, 0.1321350634098053, 0.12838445603847504, 0.12456385046243668, 0.12219297885894775, 0.11941319704055786, 0.116800457239151, 0.11469992995262146, 0.11323373764753342, 0.11144670099020004, 0.11036712676286697, 0.10902441293001175, 0.10799607634544373, 0.10653714835643768, 0.10548193752765656, 0.10435868799686432, 0.1034649983048439, 0.10238385945558548, 0.10163772851228714, 0.10103771835565567, 0.10045122355222702 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 6.883365585963475e-05, "first_to_last_ratio": 2.2497546817252747, "imbalance_abs_log_ratio": 0.8108211799277664, "last_quartile_mean": 3.059607183786284e-05, "mean": 4.691841348858361e-05, "normalized": [ 1.7359303646705315, 1.6475859007944922, 1.5536416135247637, 1.4921074478374423, 1.436565604934822, 1.343834173349448, 1.2787439414688093, 1.2483315192759759, 1.191227587726315, 1.1603086071531024, 1.0995306735006558, 1.059977946939621, 1.0583984121521266, 1.0176934649371145, 0.9754909410255406, 0.9631621800279858, 0.9242301483750994, 0.8752327792179452, 0.8454160845678106, 0.8232909667825862, 0.7863453139125789, 0.77746685672799, 0.7497386589361914, 0.7388508716017261, 0.7093180450413769, 0.6908354487677191, 0.6685471901846501, 0.6592354484095886, 0.631440334976849, 0.6235901145414114, 0.6199506557967245, 0.6139807028410055 ], "peak_layer": 1, "peak_normalized": 1.7359303646705315, "population_cv": 0.3220412433027962, "rest_mean": 4.907019663530971e-05, "spike_contrast": 0.7193528234378936, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.529878449626267e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 8.144709863699973e-05, 7.730211655143648e-05, 7.289439963642508e-05, 7.000731420703232e-05, 6.740137905580923e-05, 6.305056740529835e-05, 5.999663699185476e-05, 5.856973439222202e-05, 5.5890508519951254e-05, 5.4439839004771784e-05, 5.1588234782684594e-05, 4.9732483603293076e-05, 4.965837433701381e-05, 4.774856279254891e-05, 4.576848732540384e-05, 4.519004141911864e-05, 4.3363412260077894e-05, 4.106453343410976e-05, 3.966558142565191e-05, 3.862750600092113e-05, 3.6894074582960457e-05, 3.647751145763323e-05, 3.517654840834439e-05, 3.4665710700210184e-05, 3.328007733216509e-05, 3.241290323785506e-05, 3.136717350571416e-05, 3.0930281354812905e-05, 2.9626178729813546e-05, 2.925785884144716e-05, 2.9087101211189292e-05, 2.8807000489905477e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 0.0001734125216898974, "first_to_last_ratio": 1.495517223041696, "imbalance_abs_log_ratio": 0.4024721155990803, "last_quartile_mean": 0.00011595488103921525, "mean": 0.00013990532806928968, "normalized": [ 1.3768889945483427, 1.3340940553978193, 1.2910049686488183, 1.2527203273458163, 1.2147536525239127, 1.1769854337196524, 1.14744939519443, 1.1220955988146075, 1.0944022472959734, 1.0773022658134643, 1.0508559212107247, 1.03206209473486, 1.0208171898483585, 1.000561882500425, 0.9805161605170362, 0.9663129296065233, 0.9487424991891081, 0.931931984738623, 0.9181974345490227, 0.9075104531838204, 0.8943640941785878, 0.8862664021756573, 0.8757830774574916, 0.86790401991425, 0.8562894022841879, 0.8478155487572495, 0.8387227683262297, 0.8314011142413483, 0.8224071457687988, 0.8161875050523302, 0.8112199678685613, 0.8064334645939688 ], "peak_layer": 1, "peak_normalized": 1.3768889945483427, "population_cv": 0.16325957486062354, "rest_mean": 0.0001431148236776978, "spike_contrast": 0.8564734849544942, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001225740517838858, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0001926341064972803, 0.00018664686649572104, 0.00018061847367789596, 0.00017526224837638438, 0.0001699505082797259, 0.00016466653323732316, 0.00016053428407758474, 0.00015698715287726372, 0.00015311270544771105, 0.00015072032692842185, 0.00014702034241054207, 0.00014439098595175892, 0.00014281776384450495, 0.00013998393842484802, 0.00013717943511437625, 0.00013519232743419707, 0.00013273413060232997, 0.00013038225006312132, 0.00012846071331296116, 0.00012696554767899215, 0.0001251263020094484, 0.00012399339175317436, 0.0001225267187692225, 0.00012142439663875848, 0.00011979944974882528, 0.00011861391249112785, 0.00011734178406186402, 0.00011631744564510882, 0.00011505914153531194, 0.00011418898066040128, 0.00011349399574100971, 0.00011282433843007311 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 0.00014675117381557357, "first_to_last_ratio": 1.2676837465168667, "imbalance_abs_log_ratio": 0.23719141364257146, "last_quartile_mean": 0.00011576323686313117, "mean": 0.00013042020850662084, "normalized": [ 1.169988069421413, 1.156175152925158, 1.1417665260225645, 1.1306726331060184, 1.1183621029407895, 1.1062694829504214, 1.0940115056230622, 1.0844989956600726, 1.0711591662817939, 1.0579765491085218, 1.0443900226505625, 1.035471661013805, 1.0260498632741444, 1.013992390086173, 1.0062479316494695, 0.9977001177294099, 0.9863163473759529, 0.9769959732660659, 0.9687585669786032, 0.9602628595878113, 0.9503845997255937, 0.9426104617668664, 0.9330243108845692, 0.9259760938667744, 0.914797625470175, 0.906289421438531, 0.8956357003805738, 0.888470885240542, 0.8806216558595509, 0.8756991509238615, 0.8712858841028995, 0.8681382926882502 ], "peak_layer": 1, "peak_normalized": 1.169988069421413, "population_cv": 0.09123006364857297, "rest_mean": 0.00013202972385678786, "spike_contrast": 0.9219804606102014, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012172882561571896, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00015259008796419948, 0.00015078860451467335, 0.00014890942838974297, 0.0001474625605624169, 0.00014585701865144074, 0.00014427989663090557, 0.00014268120867200196, 0.00014144058513920754, 0.00013970080181024969, 0.0001379815221298486, 0.00013620956451632082, 0.00013504642993211746, 0.0001338176371064037, 0.00013224509893916547, 0.00013123506505507976, 0.0001301202573813498, 0.00012863558367826045, 0.00012742001854348928, 0.00012634569429792464, 0.00012523768236860633, 0.00012394935765769333, 0.00012293545296415687, 0.00012168522516731173, 0.00012076599523425102, 0.00011930809705518186, 0.00011819845531135798, 0.00011680899478960782, 0.00011587455810513347, 0.00011485085997264832, 0.00011420886585256085, 0.00011363328667357564, 0.00011322277714498341 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00030002388848515693, "first_to_last_ratio": 2.2836575149168885, "imbalance_abs_log_ratio": 0.8257783308640597, "last_quartile_mean": 0.0001313786706305109, "mean": 0.0001947244400071213, "normalized": [ 1.8998833386658438, 1.7835645331795598, 1.6713073472446418, 1.5703972875779748, 1.4780250735507643, 1.3761669788219328, 1.3066447563403325, 1.2401013786965793, 1.1762106459027857, 1.150547637187856, 1.0768735950666006, 1.0264900078881174, 1.0140784159549234, 0.9666871251601796, 0.9096841069789114, 0.8754807117769676, 0.8421876125778792, 0.8073809902734768, 0.77911667677541, 0.7607651063686129, 0.7403822030704592, 0.7308855611321243, 0.7152791523187013, 0.7043381880055872, 0.6944157299089508, 0.686864629891504, 0.680599049276497, 0.6764772714226681, 0.6702412094778135, 0.6672258960044359, 0.6635863560708457, 0.6581114274310631 ], "peak_layer": 1, "peak_normalized": 1.8998833386658438, "population_cv": 0.3592853832280775, "rest_mean": 0.00020492727344390005, "spike_contrast": 0.6813594750078023, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013962913944851608, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0003699537192005664, 0.0003473036049399525, 0.0003254443872720003, 0.00030579473241232336, 0.00028780760476365685, 0.00026797334430739284, 0.0002544356684666127, 0.0002414780465187505, 0.0002290369593538344, 0.00022403974435292184, 0.00020969360775779933, 0.00019988269195891917, 0.0001974658516701311, 0.00018823760910890996, 0.00017713772831484675, 0.00017047749133780599, 0.00016399451124016196, 0.0001572168112033978, 0.00015171305858530104, 0.00014813955931458622, 0.00014417050988413393, 0.00014232128160074353, 0.00013928233238402754, 0.0001371518592350185, 0.00013521971413865685, 0.00013374933041632175, 0.00013252926873974502, 0.00013172665785532445, 0.00013051234418526292, 0.00012992518895771354, 0.00012921648158226162, 0.000128150379168801 ] } } }, "mlp_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "afe29f3ef4d2992353f0bb59fe7c5c2c5050fca671a53184825a8d63cd83a2bc", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.0027073738165199757, 0.002722575329244137, 0.0026850115973502398, 0.0026861224323511124, 0.00271838391199708, 0.0026645169127732515, 0.0026859452482312918, 0.0027359756641089916, 0.0026767742820084095, 0.0027451682835817337, 0.0026999011170119047, 0.002663193503394723, 0.002626648638397455, 0.0026531771291047335, 0.0026517712976783514, 0.002676903735846281, 0.002670086920261383, 0.0026355870068073273, 0.0026610493659973145, 0.0026388936676084995, 0.0026469926815479994, 0.00264862016774714, 0.002641494618728757, 0.002674198243767023, 0.002557571977376938, 0.00252733426168561, 0.0025468147359788418, 0.0026856842450797558, 0.002551089273765683, 0.0025778315030038357, 0.0026001501828432083, 0.0025662274565547705 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.00015858857113926206, "first_to_last_ratio": 3.757867690304466, "imbalance_abs_log_ratio": 1.3238516928952826, "last_quartile_mean": 4.220174423608114e-05, "mean": 8.573221236929385e-05, "normalized": [ 2.5461641896458365, 2.3032502265274117, 2.0924715218207024, 1.8886722264368014, 1.7031663195050915, 1.537434754996766, 1.4223955235407997, 1.3049501874408618, 1.2055297161045848, 1.1692542365736316, 1.0769571962730058, 1.0144505847411098, 0.9913601784485884, 0.9317001386730132, 0.857757082878185, 0.8155634736467603, 0.7682612945963089, 0.7217806418748633, 0.678338533281248, 0.6511820384364709, 0.6192882306865346, 0.6053793146733001, 0.5848721334223365, 0.5718144052517429, 0.5462281883927541, 0.527515083157495, 0.5112993585545178, 0.496920907994256, 0.47722358366421574, 0.46552107689700334, 0.4601234027970099, 0.45317424906679377 ], "peak_layer": 1, "peak_normalized": 2.5461641896458365, "population_cv": 0.5686091737198813, "rest_mean": 9.231269224848668e-05, "spike_contrast": 0.5437781067692293, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.0197621021652594e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00021828828903380781, 0.00019746273756027222, 0.00017939221288543195, 0.0001619200484128669, 0.00014601621660403907, 0.000131807682919316, 0.00012194511509733275, 0.00011187626660102978, 0.00010335272963857278, 0.00010024275252362713, 9.232992306351662e-05, 8.697109296917915e-05, 8.499150135321543e-05, 7.987671415321529e-05, 7.353741239057854e-05, 6.992006092332304e-05, 6.586474046343938e-05, 6.187985127326101e-05, 5.8155463193543255e-05, 5.582727681030519e-05, 5.309295011102222e-05, 5.1900507969548926e-05, 5.0142381951445714e-05, 4.902291402686387e-05, 4.682935104938224e-05, 4.522503513726406e-05, 4.3834825191879645e-05, 4.260212881490588e-05, 4.0913433622336015e-05, 3.991015182691626e-05, 3.944739728467539e-05, 3.885163096128963e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.00021242323236947414, "first_to_last_ratio": 1.8345651540958352, "imbalance_abs_log_ratio": 0.6068074801791611, "last_quartile_mean": 0.0001157894184871111, "mean": 0.00015111673860701558, "normalized": [ 1.771707915810251, 1.6366668487510905, 1.5242909365839221, 1.4186705636795862, 1.3264449439106374, 1.2453857912414694, 1.1893415085703798, 1.1330085309323608, 1.0858463851390703, 1.062861239127963, 1.023237806768965, 0.9942126959777651, 0.978436257513136, 0.9531027445378829, 0.9232414968561398, 0.904013237249231, 0.8853704564278231, 0.8680164030054869, 0.8515402788738432, 0.8395817783841374, 0.8267435189825344, 0.8185143642831039, 0.809172703595369, 0.8007918347797284, 0.7909896391756025, 0.7821052393452608, 0.7753388667989797, 0.7682453290925442, 0.7604995795745633, 0.7549901567959042, 0.7508572350862155, 0.7467737131490535 ], "peak_layer": 1, "peak_normalized": 1.771707915810251, "population_cv": 0.2714170004285523, "rest_mean": 0.00015645500981783562, "spike_contrast": 0.7816309251520459, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012229007406858728, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0002677347220014781, 0.00024732775636948645, 0.00023034587502479553, 0.00021438486874103546, 0.00020044803386554122, 0.0001881986390799284, 0.00017972940986510366, 0.00017121655400842428, 0.00016408956435043365, 0.00016061612404882908, 0.0001546283601783216, 0.00015024218009784818, 0.00014785809617023915, 0.0001440297783119604, 0.00013951724395155907, 0.000136611532070674, 0.0001337942958343774, 0.00013117180787958205, 0.0001286819897359237, 0.00012687486014328897, 0.00012493478425312787, 0.00012369122123345733, 0.00012227953993715346, 0.00012101305037504062, 0.00011953177454415709, 0.00011818919301731512, 0.00011716668086592108, 0.00011609472858253866, 0.00011492421617731452, 0.00011409165017539635, 0.00011346709652571008, 0.00011285000800853595 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.18837904930114746, "first_to_last_ratio": 1.8345651581276443, "imbalance_abs_log_ratio": 0.6068074823768531, "last_quartile_mean": 0.10268321540206671, "mean": 0.13401183625683188, "normalized": [ 1.7717078885231814, 1.6366668119471057, 1.524290957183596, 1.418670514273678, 1.326444920903592, 1.2453858718827995, 1.1893415500975306, 1.1330085716119305, 1.0858463365841577, 1.062861212873729, 1.0232377434953237, 0.9942127340740732, 0.9784362443707608, 0.9531027271661117, 0.9232414917376033, 0.904013241653558, 0.885370477646398, 0.8680164390463408, 0.8515402681847718, 0.8395816969955481, 0.8267435345782925, 0.8185144279720658, 0.8091727254708025, 0.8007918405918669, 0.7909896328038464, 0.7821052665924377, 0.7753388466443417, 0.7682453528580236, 0.7604996016741158, 0.7549901615676585, 0.7508572336421532, 0.7467736753526057 ], "peak_layer": 1, "peak_normalized": 1.7717078885231814, "population_cv": 0.27141699472708714, "rest_mean": 0.1387458680956452, "spike_contrast": 0.7816309473985974, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.10844806432724, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.23742982745170593, 0.2193327248096466, 0.2042730301618576, 0.19011864066123962, 0.1777593195438385, 0.16689644753932953, 0.15938584506511688, 0.151836559176445, 0.1455162614583969, 0.1424359828233719, 0.13712596893310547, 0.13323627412319183, 0.13112203776836395, 0.12772704660892487, 0.1237252876162529, 0.1211484745144844, 0.11865012347698212, 0.11632447689771652, 0.11411647498607635, 0.11251388490200043, 0.11079341918230057, 0.10969062149524689, 0.10843872278928757, 0.10731558501720428, 0.10600197315216064, 0.10481136292219162, 0.10390458256006241, 0.1029539704322815, 0.10191594809293747, 0.10117761790752411, 0.10062375664710999, 0.10007651150226593 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 6.873187476230669e-05, "first_to_last_ratio": 2.248757500145603, "imbalance_abs_log_ratio": 0.8103778415280068, "last_quartile_mean": 3.0564378221242805e-05, "mean": 4.685761763312257e-05, "normalized": [ 1.7400006587426948, 1.6405194725375354, 1.5551306973978087, 1.4944078341288562, 1.4327724019957897, 1.3414187773842097, 1.2804088050749736, 1.2499327782691996, 1.1927955171655855, 1.1608931065609767, 1.0981756972550076, 1.0577423904261343, 1.0542786038595535, 1.0218640087393278, 0.9775807389332813, 0.9570972402588515, 0.9209982861589985, 0.8815092249082567, 0.8478232134430647, 0.8204952153707635, 0.7871780668180614, 0.7759097745574741, 0.7506963524238334, 0.7421155343193898, 0.7072927325438048, 0.6891814936654451, 0.6695399899996352, 0.6616343983211914, 0.6345111343049281, 0.6212687927046295, 0.6200231913288072, 0.6148038704019305 ], "peak_layer": 1, "peak_normalized": 1.7400006587426948, "population_cv": 0.3217536161653951, "rest_mean": 4.900405669845727e-05, "spike_contrast": 0.719671983430414, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.526684668031521e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 8.153228554874659e-05, 7.687083416385576e-05, 7.286971958819777e-05, 7.002439087955281e-05, 6.71363013680093e-05, 6.285568815656006e-05, 5.999690620228648e-05, 5.8568872191244736e-05, 5.5891556257847697e-05, 5.439668530016206e-05, 5.145789691596292e-05, 4.9563288484932855e-05, 4.9400983698433265e-05, 4.7882112994557247e-05, 4.580710447044112e-05, 4.484729652176611e-05, 4.3155785533599555e-05, 4.130542220082134e-05, 3.972697595600039e-05, 3.844645107164979e-05, 3.688528886414133e-05, 3.635728353401646e-05, 3.517584264045581e-05, 3.477376594673842e-05, 3.314205241622403e-05, 3.229340290999971e-05, 3.1373048841487616e-05, 3.10026116494555e-05, 2.9731680115219206e-05, 2.9111175535945222e-05, 2.9052809622953646e-05, 2.8808244678657502e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 0.0001719075116852764, "first_to_last_ratio": 1.4894635014227988, "imbalance_abs_log_ratio": 0.39841598895878805, "last_quartile_mean": 0.00011541572621354135, "mean": 0.00013894635094402474, "normalized": [ 1.3738985779819257, 1.3296082759737424, 1.2883384004275333, 1.2499940655466777, 1.2130730176151365, 1.1755503979222366, 1.147409009170091, 1.11990595257071, 1.0929037170303153, 1.075598370036652, 1.0512762916941902, 1.0312594730951, 1.0194418974669357, 1.0018996501730997, 0.9803460168062212, 0.9648004557293595, 0.9493010807852901, 0.9344788929945884, 0.9198953859218769, 0.908001147597105, 0.8952913731815082, 0.886953676542665, 0.8771894463102521, 0.8683888934041172, 0.8579399877904575, 0.8482799643655919, 0.8408636339472488, 0.8330870307470335, 0.82434015220157, 0.8181187963300758, 0.8135579418994486, 0.809009026741244 ], "peak_layer": 1, "peak_normalized": 1.3738985779819257, "population_cv": 0.16161423472023242, "rest_mean": 0.0001421073115866252, "spike_contrast": 0.8576417505420824, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012187716347398237, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0001908981939777732, 0.0001847442181315273, 0.00017900991952046752, 0.00017368211410939693, 0.00016855206922627985, 0.00016333843814209104, 0.00015942829486448318, 0.00015560684551019222, 0.0001518549834145233, 0.00014945046859793365, 0.00014607100456487387, 0.0001432897406630218, 0.00014164773165248334, 0.00013921030040364712, 0.000136215501697734, 0.0001340555027127266, 0.0001319019211223349, 0.00012984243221580982, 0.0001278161071240902, 0.00012616344611160457, 0.00012439746933523566, 0.00012323897681199014, 0.00012188227265141904, 0.00012065946793882176, 0.00011920763063244522, 0.00011786540562752634, 0.00011683493357850239, 0.00011575440294109285, 0.0001145390560850501, 0.00011367462138878182, 0.00011304090730845928, 0.00011240885214647278 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 0.00014552221000485588, "first_to_last_ratio": 1.262697055715921, "imbalance_abs_log_ratio": 0.2332499537229619, "last_quartile_mean": 0.00011524712863320019, "mean": 0.00012953460168318998, "normalized": [ 1.1676213631634658, 1.1525055688027706, 1.1391580058081985, 1.1289307982409096, 1.1166731577821778, 1.1054001772932092, 1.094813144753513, 1.0822854389705017, 1.0679308603491235, 1.0564894822224282, 1.0456838360716472, 1.0337072702557388, 1.0234945545465333, 1.0158819477934218, 1.0051772952892828, 0.9951467950359763, 0.9866643397779161, 0.9787559418439278, 0.970448736962688, 0.9607908684400951, 0.9509939230158972, 0.9428061355388626, 0.934924980051271, 0.9261034265480621, 0.9170582603044295, 0.9058562786860701, 0.8981778407016658, 0.8907564927732574, 0.8831168120837269, 0.8778216107141017, 0.8745192645108896, 0.8703053916682415 ], "peak_layer": 1, "peak_normalized": 1.1676213631634658, "population_cv": 0.08967236486753827, "rest_mean": 0.00013110875065817877, "spike_contrast": 0.9231588022206524, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012103419721825049, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00015124736819416285, 0.00014928934979252517, 0.000147560378536582, 0.00014623560127802193, 0.00014464781270362437, 0.00014318757166620344, 0.00014181618462316692, 0.00014019341324456036, 0.0001383339986205101, 0.00013685194426216185, 0.00013545223919209093, 0.00013390085950959474, 0.00013257795944809914, 0.00013159186346456409, 0.00013020524056628346, 0.00012890594371128827, 0.00012780717224813998, 0.00012678276107180864, 0.0001257066905964166, 0.0001244556624442339, 0.00012318661902099848, 0.0001221260172314942, 0.00012110513489460573, 0.0001199624384753406, 0.00011879077646881342, 0.00011733973224181682, 0.00011634510883595794, 0.00011538378748809919, 0.0001143941844929941, 0.00011370827269274741, 0.00011328050459269434, 0.0001127346622524783 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00029741139769612346, "first_to_last_ratio": 2.278072415841521, "imbalance_abs_log_ratio": 0.8233296537390045, "last_quartile_mean": 0.00013055396993877366, "mean": 0.00019328556300024502, "normalized": [ 1.889398576482247, 1.780233894901915, 1.6772594716902607, 1.566224958959574, 1.4762232610465842, 1.375571144634086, 1.3083686481730985, 1.2364401527267181, 1.173148897934763, 1.1508864184295666, 1.080457040307468, 1.0318582947152708, 1.0081957004611959, 0.9698994715426978, 0.9066127339785177, 0.8754550048258108, 0.8446146862114313, 0.8115724449181009, 0.7806677558079268, 0.7613480996239201, 0.7403134026279838, 0.7294052762839086, 0.7160186976339193, 0.7062575707090604, 0.6942777320753531, 0.6866005528973866, 0.6829938476675228, 0.6753403086482458, 0.6715967314167978, 0.6686860556999406, 0.6640286432895989, 0.6600445236791291 ], "peak_layer": 1, "peak_normalized": 1.889398576482247, "population_cv": 0.3579154260008552, "rest_mean": 0.00020340604748559632, "spike_contrast": 0.681567477924495, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013863494677934796, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0003651934675872326, 0.0003440935106482357, 0.00032419004128314555, 0.0003027286729775369, 0.0002853326441254467, 0.00026587804313749075, 0.00025288877077400684, 0.00023898603103589267, 0.00022675274522043765, 0.00022244972933549434, 0.0002088367473334074, 0.00019944331143051386, 0.00019486967357806861, 0.00018746756541077048, 0.00017523515271022916, 0.00016921281348913908, 0.0001632518251426518, 0.00015686523693148047, 0.00015089180669747293, 0.00014715759607497603, 0.00014309189282357693, 0.00014098350948188454, 0.00013839607709087431, 0.0001365093921776861, 0.0001341938623227179, 0.00013270997442305088, 0.00013201285037212074, 0.00013053353177383542, 0.0001298099523410201, 0.00012924736074637622, 0.00012834715016651899, 0.00012757707736454904 ] } } }, "post_attention_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "31853ffb07b24cc59e774bb20656840b4358b3080b42516d380f2dc0f46404c8", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.0022265110164880753, 0.0040087271481752396, 0.005379301495850086, 0.006410556845366955, 0.0027253651060163975, 0.004838948138058186, 0.006175771821290255, 0.00742313964292407, 0.0030937574338167906, 0.005150026176124811, 0.007019025273621082, 0.008143599145114422, 0.004255065228790045, 0.0066224560141563416, 0.008129785768687725, 0.010099578648805618, 0.0041999719105660915, 0.00719039561226964, 0.009030314162373543, 0.010736005380749702, 0.005131533369421959, 0.00787449348717928, 0.009912566281855106, 0.012113703414797783, 0.0059690922498703, 0.00876442901790142, 0.010806876234710217, 0.012353328987956047, 0.006502739619463682, 0.009470963850617409, 0.012123330496251583, 0.01384514570236206 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.00015962236011546338, "first_to_last_ratio": 3.7698422272252903, "imbalance_abs_log_ratio": 1.3270331510418396, "last_quartile_mean": 4.2341920561739244e-05, "mean": 8.61727822893954e-05, "normalized": [ 2.542734798705971, 2.3221448837042376, 2.101145461274622, 1.8911858524935854, 1.7004204056817789, 1.5354142426015858, 1.4229001770142031, 1.3028733810328659, 1.2049412416934622, 1.1705258152816296, 1.0773908232649334, 1.0133220774376726, 0.990609360100461, 0.9272534949358001, 0.8550385909364457, 0.8185651575045715, 0.7703972051466339, 0.7173947009730917, 0.6760256695667757, 0.6514523109028573, 0.618419077603279, 0.6053620388084041, 0.5838930841859182, 0.5697043695032278, 0.5457802700901755, 0.5276149005446632, 0.5104701332743111, 0.49517988582787786, 0.47527562953524777, 0.46579059316028215, 0.45896435764104826, 0.45181000957238127 ], "peak_layer": 1, "peak_normalized": 2.542734798705971, "population_cv": 0.5705202801130907, "rest_mean": 9.280119640472414e-05, "spike_contrast": 0.542873885449774, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.03793460666202e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00021911453222855926, 0.00020010568550787866, 0.00018106155039276928, 0.00016296874673571438, 0.00014652995741926134, 0.00013231091725174338, 0.0001226152671733871, 0.00011227222421439365, 0.00010383313929196447, 0.00010086746624438092, 9.284176485380158e-05, 8.732078276807442e-05, 8.536356472177431e-05, 7.99040135461837e-05, 7.368105434579775e-05, 7.053803710732609e-05, 6.638727063545957e-05, 6.181989738252014e-05, 5.825501284562051e-05, 5.613745815935545e-05, 5.329089253791608e-05, 5.216573117650114e-05, 5.0315691623836756e-05, 4.9093010602518916e-05, 4.703140439232811e-05, 4.546604395727627e-05, 4.398863165988587e-05, 4.267102849553339e-05, 4.095582335139625e-05, 4.013847137684934e-05, 3.955023566959426e-05, 3.893372559105046e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.00021442477373057045, "first_to_last_ratio": 1.8431254562763404, "imbalance_abs_log_ratio": 0.6114627481432037, "last_quartile_mean": 0.00011633759004325839, "mean": 0.00015221248349917005, "normalized": [ 1.7740805953138137, 1.6484457246979003, 1.5301434809496548, 1.4217241600875643, 1.3264958046529274, 1.2454577490392524, 1.1897861330408546, 1.1336265611690741, 1.086914920172498, 1.064301358569379, 1.0227209980571423, 0.9941753882589759, 0.9788979908211966, 0.9511107967643203, 0.9228143274453928, 0.9052447611165039, 0.8846530997418666, 0.8652930871753776, 0.8497381552743479, 0.8388656494639406, 0.8256334792987466, 0.8176309647342147, 0.8076951735882674, 0.8000661307359944, 0.7892605467734257, 0.78144613040264, 0.7731219607566042, 0.7664988488799842, 0.7584935618388989, 0.7529639051546407, 0.7485195296370439, 0.7441790263875565 ], "peak_layer": 1, "peak_normalized": 1.7740805953138137, "population_cv": 0.2736411578809574, "rest_mean": 0.00015762286892384981, "spike_contrast": 0.7803207938393821, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012299640220589936, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00027003721334040165, 0.00025091401766985655, 0.00023290693934541196, 0.0002164041652576998, 0.000201909220777452, 0.0001895742170745507, 0.00018110030214302242, 0.00017255211423616856, 0.00016544201935175806, 0.00016199995297938585, 0.00015567090304102749, 0.00015132590488065034, 0.0001490004942752421, 0.00014477093645837158, 0.00014046386058907956, 0.00013778955326415598, 0.0001346552453469485, 0.00013170840975362808, 0.00012934075493831187, 0.00012768582382705063, 0.00012567172234412283, 0.00012445363972801715, 0.00012294128828216344, 0.00012178005272289738, 0.00012013530795229599, 0.00011894585622940212, 0.00011767881369451061, 0.00011667069338727742, 0.00011545218876563013, 0.00011461050598882139, 0.00011393401655368507, 0.00011327333777444437 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.190154030919075, "first_to_last_ratio": 1.84312545379361, "imbalance_abs_log_ratio": 0.6114627467961817, "last_quartile_mean": 0.10316933691501617, "mean": 0.13498355145566165, "normalized": [ 1.774080628234443, 1.6484456620184311, 1.5301434020504485, 1.4217242444591272, 1.326495766996536, 1.2454576834375484, 1.189786122599316, 1.133626627255238, 1.0869149569673349, 1.0643014004698326, 1.0227209907477024, 0.9941753870542451, 0.9788979589698585, 0.9511107512164827, 0.9228143032197887, 0.9052447972745712, 0.8846531331847195, 0.8652931173486506, 0.8497382179112226, 0.8388656682549475, 0.8256334805754935, 0.8176309131099365, 0.8076951535479161, 0.8000661540388709, 0.7892605573866218, 0.7814461538045975, 0.7731219580171443, 0.7664988542586116, 0.7584935821787124, 0.7529638601652359, 0.7485195164462104, 0.7441789968002049 ], "peak_layer": 1, "peak_normalized": 1.774080628234443, "population_cv": 0.27364115345546325, "rest_mean": 0.13978153549962574, "spike_contrast": 0.7803207857775153, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.10907443761825561, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.2394717037677765, 0.22251304984092712, 0.2065441906452179, 0.19190938770771027, 0.1790551096200943, 0.16811630129814148, 0.16060155630111694, 0.1530209481716156, 0.14671564102172852, 0.1436631828546524, 0.13805051147937775, 0.13419732451438904, 0.13213512301445007, 0.1283843070268631, 0.1245647519826889, 0.12219315767288208, 0.11941362172365189, 0.11680033802986145, 0.11470068246126175, 0.1132330670952797, 0.11144693940877914, 0.11036672443151474, 0.10902556031942368, 0.10799577087163925, 0.10653719305992126, 0.10548237711191177, 0.1043587476015091, 0.10346473753452301, 0.10238415747880936, 0.10163773596286774, 0.10103782266378403, 0.10045192390680313 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 6.883367768750759e-05, "first_to_last_ratio": 2.249759357551583, "imbalance_abs_log_ratio": 0.810823258297238, "last_quartile_mean": 3.059601795030176e-05, "mean": 4.691846504556452e-05, "normalized": [ 1.7359059710084281, 1.6475789727938472, 1.5536456441204052, 1.4921194549590373, 1.4365657321885223, 1.3438513058553838, 1.2787343947807137, 1.2483299149152907, 1.1912354282513808, 1.1603083401306369, 1.099547144002948, 1.059972440022979, 1.0583955432739494, 1.0176800955770247, 0.9754963047746087, 0.9631595708780456, 0.9242400656751482, 0.8752301116137658, 0.8454245377074081, 0.8232853322610043, 0.7863494898171388, 0.7774642190168813, 0.7497486129019063, 0.7388483538626245, 0.709315327139865, 0.6908343794807076, 0.6685469983108738, 0.6592365073812129, 0.6314341746588822, 0.6235898169926835, 0.6199474545603838, 0.6139783610863121 ], "peak_layer": 1, "peak_normalized": 1.7359059710084281, "population_cv": 0.3220406003070692, "rest_mean": 4.9070239684725595e-05, "spike_contrast": 0.7193541792517962, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.52988819940947e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 8.144604362314567e-05, 7.730187644483522e-05, 7.28946688468568e-05, 7.000795449130237e-05, 6.740145909134299e-05, 6.305144052021205e-05, 5.999625500408001e-05, 5.8569723478285596e-05, 5.589093780145049e-05, 5.4439886298496276e-05, 5.158906424185261e-05, 4.973227987647988e-05, 4.9658294301480055e-05, 4.774798799189739e-05, 4.576878927764483e-05, 4.51899686595425e-05, 4.3363925215089694e-05, 4.1064453398576006e-05, 3.9666021621087566e-05, 3.86272840842139e-05, 3.689431105158292e-05, 3.6477427784120664e-05, 3.5177054087398574e-05, 3.466563066467643e-05, 3.3279986382694915e-05, 3.2412888685939834e-05, 3.136719897156581e-05, 3.093036502832547e-05, 2.9625922252307646e-05, 2.9257877031341195e-05, 2.9086982976878062e-05, 2.8806922273361124e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 0.00017341253987979144, "first_to_last_ratio": 1.4955148931299893, "imbalance_abs_log_ratio": 0.4024705576674927, "last_quartile_mean": 0.00011595507385209203, "mean": 0.00013990553861731314, "normalized": [ 1.3768818258178377, 1.3340879911950096, 1.2910054180628048, 1.2527236427111486, 1.2147542166907819, 1.1769889670731195, 1.1474411155803916, 1.1220953663134672, 1.0944075691302455, 1.0773007485615413, 1.0508678613623372, 1.0320596054419082, 1.0208161736522385, 1.000559648637946, 0.9805197815181176, 0.9663128275349431, 0.948744191771126, 0.9319305822463463, 0.9182019814348666, 0.9075038868226692, 0.8943650364968407, 0.886262676120959, 0.8757907045344768, 0.8679002174812178, 0.856288373659771, 0.8478178092769747, 0.8387215581135022, 0.8313975747674917, 0.8224084044020752, 0.8161864847722404, 0.8112200992010112, 0.8064376596145924 ], "peak_layer": 1, "peak_normalized": 1.3768818258178377, "population_cv": 0.16325888721442564, "rest_mean": 0.0001431150389921472, "spike_contrast": 0.8564734877369171, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001225742365932092, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0001926333934534341, 0.00018664629897102714, 0.00018061880837194622, 0.0001752629759721458, 0.00016995084297377616, 0.0001646672753849998, 0.00016053336730692536, 0.00015698735660407692, 0.00015311368042603135, 0.00015072034148033708, 0.00014702223415952176, 0.00014439085498452187, 0.0001428178366040811, 0.00013998383656144142, 0.00013718014815822244, 0.00013519251660909504, 0.00013273456715978682, 0.00013038225006312132, 0.00012846154277212918, 0.00012696482008323073, 0.00012512662215158343, 0.0001239930570591241, 0.00012252797023393214, 0.000121424047392793, 0.00011979948612861335, 0.00011861440725624561, 0.00011734179133782163, 0.0001163171255029738, 0.00011505949078127742, 0.00011418900976423174, 0.00011349418491590768, 0.00011282509512966499 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 0.00014675183047074825, "first_to_last_ratio": 1.2676545313577883, "imbalance_abs_log_ratio": 0.23716836728277432, "last_quartile_mean": 0.00011576642282307148, "mean": 0.00013041823922321782, "normalized": [ 1.1700698937843759, 1.1560440994564782, 1.1417398043557874, 1.130463200949706, 1.118403537277814, 1.1065304334184665, 1.0939881912841023, 1.0846815122144253, 1.0709468270677047, 1.0579907390333845, 1.0442838370166418, 1.035550673160169, 1.025959021745254, 1.0140203095307772, 1.0062047700154308, 0.9976631870190885, 0.9861765922450596, 0.9770016877798172, 0.9687702939454803, 0.9602422119882037, 0.9504995944091904, 0.9425883202531964, 0.9330014667281973, 0.9259385264566372, 0.9147443798098084, 0.906202796813466, 0.8958065504175365, 0.8885655303249116, 0.8806194435723184, 0.8757905905296257, 0.8713783728577705, 0.8681336045391749 ], "peak_layer": 1, "peak_normalized": 1.1700698937843759, "population_cv": 0.09121429133712369, "rest_mean": 0.0001320278307604086, "spike_contrast": 0.921975648780332, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001217264449223876, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0001525984553154558, 0.0001507692359155044, 0.000148903694935143, 0.00014743302017450333, 0.00014586022007279098, 0.00014431175077334046, 0.00014267601363826543, 0.00014146225294098258, 0.000139670999487862, 0.00013798128929920495, 0.00013619365927297622, 0.00013505469541996717, 0.00013380376913119107, 0.00013224674330558628, 0.00013122745440341532, 0.00013011347618885338, 0.00012861541472375393, 0.00012741883983835578, 0.0001263453159481287, 0.0001252330985153094, 0.0001239624834852293, 0.00012293070903979242, 0.00012168040848337114, 0.00012075927224941552, 0.00011929935135412961, 0.00011818537313956767, 0.00011682951299007982, 0.00011588515189941972, 0.00011484883725643158, 0.00011421906674513593, 0.00011364363308530301, 0.00011322045611450449 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00030004180916876066, "first_to_last_ratio": 2.2837506327398005, "imbalance_abs_log_ratio": 0.8258191057718037, "last_quartile_mean": 0.00013138116082700435, "mean": 0.00019473525253488333, "normalized": [ 1.9000096511818072, 1.783700741635724, 1.6708066910504589, 1.570366884717585, 1.4783058799917985, 1.376185321596543, 1.3064973297630273, 1.2402700043336092, 1.1767953849206394, 1.1505341943958838, 1.0771542571420398, 1.026098162637436, 1.013898063759585, 0.9669473025647412, 0.9095398902231254, 0.8755205777387188, 0.842301214161826, 0.8073124727024382, 0.7791991818813049, 0.760574607765185, 0.7402055397447304, 0.7308143413325056, 0.7152667122155245, 0.7043714155678553, 0.6942467749765324, 0.6868749151283189, 0.680655564574882, 0.676363713518914, 0.6701061029149729, 0.6672276319278244, 0.6636177363814285, 0.6582317375530352 ], "peak_layer": 1, "peak_normalized": 1.9000096511818072, "population_cv": 0.35930746026536914, "rest_mean": 0.00020494150898971216, "spike_contrast": 0.6812747128050888, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013962146767880766, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0003699988592416048, 0.0003473494143690914, 0.0003253649629186839, 0.0003058057918678969, 0.0002878782688640058, 0.0002679917961359024, 0.0002544210874475539, 0.00024152429250534624, 0.00022916354646440595, 0.000224049566895701, 0.00020975990628357977, 0.00019981748482678086, 0.000197441695490852, 0.0001882987271528691, 0.00017711948021315038, 0.00017049472080543637, 0.00016402573965024203, 0.00015721219824627042, 0.00015173754945863038, 0.00014811068831477314, 0.00014414411270990968, 0.0001423153153154999, 0.0001392876438330859, 0.00013716594548895955, 0.00013519432104658335, 0.00013375876005738974, 0.00013254763325676322, 0.0001317118585575372, 0.00013049328117631376, 0.00012993274140171707, 0.00012922976748086512, 0.00012818092363886535 ] } } }, "post_mlp_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "7fd9c911e93175896ad618842c120e77b3f017c17b7eec9c2bfb01fc6dbc8ac2", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.0034885562490671873, 0.004831995815038681, 0.005952076520770788, 0.006925336085259914, 0.00385080324485898, 0.005542940925806761, 0.006729288958013058, 0.007927131839096546, 0.004081308841705322, 0.005811207927763462, 0.007534808944910765, 0.008581922389566898, 0.00499920267611742, 0.007102519739419222, 0.008542952127754688, 0.010447698645293713, 0.005041283555328846, 0.007636935915797949, 0.009387043304741383, 0.011020516976714134, 0.005807018838822842, 0.008367395959794521, 0.01024632342159748, 0.012378672137856483, 0.006453148555010557, 0.009127611294388771, 0.011037957854568958, 0.012611051090061665, 0.007010402157902718, 0.009794782847166061, 0.012445803731679916, 0.014087481424212456 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.00015858902861509705, "first_to_last_ratio": 3.757869257587208, "imbalance_abs_log_ratio": 1.3238521099622325, "last_quartile_mean": 4.220184837322449e-05, "mean": 8.57325238712292e-05, "normalized": [ 2.5462257183789148, 2.3032637538343064, 2.0924289533379263, 1.8886492391629932, 1.7031256747388024, 1.5374176267930122, 1.4224231993460965, 1.3049597038569594, 1.2055618292025094, 1.169245405308266, 1.0769812048488787, 1.0144334047898542, 0.9913512297333369, 0.9317090592952362, 0.8577499774884012, 0.81555804918997, 0.7682725064221458, 0.7217807351250844, 0.6783324192661732, 0.6511800118936032, 0.6192916242842709, 0.6053728292385717, 0.5848720876072855, 0.5718124973482859, 0.5462302773879167, 0.5275173250132701, 0.5112998770983732, 0.49691833866078555, 0.47721658788927634, 0.46551989467389665, 0.46012635628943105, 0.45317260249616614 ], "peak_layer": 1, "peak_normalized": 2.5462257183789148, "population_cv": 0.5686111305901927, "rest_mean": 9.231303704803361e-05, "spike_contrast": 0.5437775023084314, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.019775271648541e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00021829435718245804, 0.00019746461475733668, 0.0001793892151908949, 0.0001619186659809202, 0.00014601326256524771, 0.00013180669338908046, 0.00012194793089292943, 0.00011187748896190897, 0.00010335585830034688, 0.00010024235962191597, 9.233231685357168e-05, 8.69699360919185e-05, 8.499104296788573e-05, 7.987776916706935e-05, 7.353707042057067e-05, 6.991984992055222e-05, 6.586594099644572e-05, 6.188008410390466e-05, 5.8155150327365845e-05, 5.582730591413565e-05, 5.309343396220356e-05, 5.190014053368941e-05, 5.014256021240726e-05, 4.90229285787791e-05, 4.682970029534772e-05, 4.522539165918715e-05, 4.383502891869284e-05, 4.2602063331287354e-05, 4.0912982512963936e-05, 3.991019548266195e-05, 3.944779382436536e-05, 3.885163096128963e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.00021242368711682502, "first_to_last_ratio": 1.8345658968465297, "imbalance_abs_log_ratio": 0.6068078850437921, "last_quartile_mean": 0.0001157896194854402, "mean": 0.0001511171253696375, "normalized": [ 1.7717451736709768, 1.636674793183293, 1.5242755762025908, 1.4186515254985965, 1.3264266232459345, 1.2453738409548603, 1.1893512719369177, 1.1330135273991528, 1.085860457802012, 1.0628542818802846, 1.023252039669589, 0.9942070699734702, 0.9784308644775869, 0.9531083940389613, 0.9232390376562555, 0.9040072643238835, 0.8853775311228679, 0.8680181295617843, 0.8515367513400655, 0.8395795332993111, 0.8267439067367663, 0.8185133286645813, 0.8091730400239778, 0.800791325997512, 0.7909899739933552, 0.7821072339266707, 0.7753403490721149, 0.7682422073314642, 0.7604971517068638, 0.7549863467424895, 0.7508596466770359, 0.7467718018887739 ], "peak_layer": 1, "peak_normalized": 1.7717451736709768, "population_cv": 0.2714186609524326, "rest_mean": 0.00015645541295978345, "spike_contrast": 0.7816308178118705, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012229037238284946, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0002677410375326872, 0.0002473295899108052, 0.00023034414334688336, 0.00021438254043459892, 0.00020044577831868082, 0.00018819731485564262, 0.00017973134526982903, 0.000171217747265473, 0.0001640921109355986, 0.00016061548376455903, 0.00015463090676348656, 0.00015024171443656087, 0.0001478576596127823, 0.00014403100067283958, 0.00013951722939964384, 0.00013661097909789532, 0.00013379570737015456, 0.0001311724045081064, 0.0001286817860091105, 0.00012687484559137374, 0.0001249351626029238, 0.00012369138130452484, 0.00012227990373503417, 0.00012101328320568427, 0.00011953213106608018, 0.0001181897969217971, 0.0001171672047348693, 0.00011609455395955592, 0.00011492414341773838, 0.0001140913664130494, 0.00011346775136189535, 0.00011285000800853595 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.18837944976985455, "first_to_last_ratio": 1.8345658800697515, "imbalance_abs_log_ratio": 0.6068078758989703, "last_quartile_mean": 0.10268339328467846, "mean": 0.1340121782850474, "normalized": [ 1.7717451751532867, 1.6366747548098712, 1.5242756140248972, 1.4186514377480737, 1.3264266357207033, 1.2453739091726768, 1.1893513017871604, 1.1330134634090196, 1.0858604665419038, 1.0628542749024097, 1.0232520332438115, 0.9942070832344971, 0.9784308561813967, 0.9531084116998004, 0.9232390798282929, 0.9040072094634526, 0.8853775581660035, 0.8680181154203164, 0.8515367049614931, 0.8395794986042229, 0.8267439263823189, 0.8185133396785964, 0.809173050926764, 0.8007913534932437, 0.7909900046670136, 0.782107217826699, 0.7753403703776178, 0.7682422246066585, 0.7604971603471317, 0.7549863443945886, 0.7508596538022533, 0.7467717694238248 ], "peak_layer": 1, "peak_normalized": 1.7717451751532867, "population_cv": 0.27141865758333467, "rest_mean": 0.13874622461972413, "spike_contrast": 0.7816308398662988, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.10844832807779312, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.23743543028831482, 0.21933434903621674, 0.20427149534225464, 0.19011656939983368, 0.17775732278823853, 0.16689527034759521, 0.15938755869865417, 0.15183760225772858, 0.14551852643489838, 0.14243541657924652, 0.13712823390960693, 0.1332358568906784, 0.1311216503381729, 0.12772813439369202, 0.1237252801656723, 0.12114797532558441, 0.1186513751745224, 0.1163249984383583, 0.11411628872156143, 0.11251387745141983, 0.11079375445842743, 0.10969075560569763, 0.10843904316425323, 0.107315793633461, 0.10600229352712631, 0.104811891913414, 0.10390505194664001, 0.10295381397008896, 0.1019158810377121, 0.10117736458778381, 0.10062433779239655, 0.10007651150226593 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 6.873207303215167e-05, "first_to_last_ratio": 2.248757814140914, "imbalance_abs_log_ratio": 0.8103779811585758, "last_quartile_mean": 3.0564462122129044e-05, "mean": 4.6857780716891284e-05, "normalized": [ 1.7400119939228862, 1.6405409364325143, 1.555121247708426, 1.4944054279879166, 1.4327475398601994, 1.3414100714975594, 1.2804101716466747, 1.2499370459040815, 1.192804641985012, 1.1608858830017554, 1.098191983597808, 1.057744066133373, 1.0542717513613309, 1.021869458338179, 0.9775749297615711, 0.9570927445976433, 0.9210091333285576, 0.8815125232807631, 0.847820728512489, 0.8204970180441978, 0.7871838673851995, 0.7759041238100245, 0.7506938949785397, 0.7421070509174572, 0.7072987335050381, 0.6891811912847667, 0.6695407652827761, 0.6616333377893114, 0.6345027924930273, 0.6212629426015038, 0.6200302724102036, 0.614801730639213 ], "peak_layer": 1, "peak_normalized": 1.7400119939228862, "population_cv": 0.32175451551289735, "rest_mean": 4.9004235565748616e-05, "spike_contrast": 0.7196709452950106, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.5266924533061685e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 8.153310045599937e-05, 7.687210745643824e-05, 7.28695304132998e-05, 7.002452184678987e-05, 6.713537004543468e-05, 6.28554989816621e-05, 5.9997179050697014e-05, 5.8569276006892323e-05, 5.589217835222371e-05, 5.439653614303097e-05, 5.1458839152473956e-05, 4.9563539505470544e-05, 4.9400834541302174e-05, 4.788253500009887e-05, 4.58069916930981e-05, 4.4847241952084005e-05, 4.315644400776364e-05, 4.130572051508352e-05, 3.972699778387323e-05, 3.8446669350378215e-05, 3.688568904181011e-05, 3.635714529082179e-05, 3.5175849916413426e-05, 3.477348946034908e-05, 3.3142448955914006e-05, 3.2293501135427505e-05, 3.13731943606399e-05, 3.1002669857116416e-05, 2.9731392714893445e-05, 2.9111002731951885e-05, 2.905324254243169e-05, 2.8808244678657502e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 0.00017190765356644988, "first_to_last_ratio": 1.4894621368061003, "imbalance_abs_log_ratio": 0.3984150727783497, "last_quartile_mean": 0.00011541592721187044, "mean": 0.00013894661265112518, "normalized": [ 1.3739040544627474, 1.3296102750467884, 1.2883342981472405, 1.2499918159027226, 1.2130627732820631, 1.175544413473833, 1.1474083142393565, 1.1199112790671786, 1.092909618037032, 1.0755929927697745, 1.0512861461262244, 1.0312571117882559, 1.0194388253067554, 1.0019061415077835, 0.980344379776871, 0.9647961249933756, 0.9493075664595542, 0.9344817410273759, 0.9198929201794295, 0.9079992279088673, 0.8952920956888009, 0.8869530532606651, 0.8771902552778059, 0.8683889858357474, 0.8579407806493952, 0.8482823463732029, 0.8408658204630546, 0.8330843095877946, 0.8243382853592501, 0.818115422616924, 0.8135611224181157, 0.8090075029660142 ], "peak_layer": 1, "peak_normalized": 1.3739040544627474, "population_cv": 0.16161403917150835, "rest_mean": 0.00014210757001786045, "spike_contrast": 0.8576421569620596, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012187744287075474, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00019089931447524577, 0.00018474484386388212, 0.00017900968668982387, 0.00017368212866131216, 0.0001685509632807225, 0.00016333791427314281, 0.00015942849859129637, 0.00015560787869617343, 0.00015185608936008066, 0.00014945000293664634, 0.00014607264893129468, 0.0001432896824553609, 0.00014164757158141583, 0.0001392114645568654, 0.00013621553080156446, 0.0001340551534667611, 0.00013190307072363794, 0.00012984307250007987, 0.0001278160052606836, 0.00012616341700777411, 0.0001243978040292859, 0.00012323912233114243, 0.00012188261462142691, 0.00012065970804542303, 0.00011920796532649547, 0.00011786595860030502, 0.00011683545744745061, 0.00011575424287002534, 0.00011453901242930442, 0.00011367436673026532, 0.00011304156214464456, 0.00011240885214647278 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 0.00014552358334185556, "first_to_last_ratio": 1.262638843036328, "imbalance_abs_log_ratio": 0.2332038508028458, "last_quartile_mean": 0.00011525352965691127, "mean": 0.00012953498708156985, "normalized": [ 1.1676304712442283, 1.1525836984023852, 1.1391521450612143, 1.1288261090295841, 1.1167600441451297, 1.1054423860149858, 1.094731137320412, 1.082319740349084, 1.067957453000188, 1.0563935463552665, 1.04560972624621, 1.0337959762176818, 1.0235535208863231, 1.0159040893770892, 1.0050628637064216, 0.9950792389307415, 0.9865814183778941, 0.9786230528286792, 0.9704418054120744, 0.9608381133357455, 0.9508407831226554, 0.9429749854457838, 0.9346839821902475, 0.9262276149685831, 0.9170542960948607, 0.9059488475652709, 0.8984648923618161, 0.8908037775288885, 0.8831483920181569, 0.8778579808382188, 0.8744051093280881, 0.8703028022960919 ], "peak_layer": 1, "peak_normalized": 1.1676304712442283, "population_cv": 0.0896642712448568, "rest_mean": 0.00013110964479252557, "spike_contrast": 0.9231344927669946, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012103183544240892, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00015124899800866842, 0.00014929991448298097, 0.00014756005839444697, 0.00014622247545048594, 0.00014465949789155275, 0.00014319346519187093, 0.00014180598373059183, 0.00014019827358424664, 0.00013833785487804562, 0.00013683992438018322, 0.0001354430423816666, 0.00013391274842433631, 0.0001325859921053052, 0.00013159512309357524, 0.00013019080506637692, 0.00012889757636003196, 0.00012779681128449738, 0.00012676592450588942, 0.00012570616672746837, 0.00012446215259842575, 0.00012316714855842292, 0.00012214825255796313, 0.00012107427755836397, 0.00011997888213954866, 0.00011879061639774591, 0.00011735207226593047, 0.0001163826382253319, 0.0001153902558144182, 0.00011439861555118114, 0.00011371332220733166, 0.00011326605454087257, 0.0001127346622524783 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00029742828155576717, "first_to_last_ratio": 2.2781890121807993, "imbalance_abs_log_ratio": 0.8233808344454435, "last_quartile_mean": 0.00013055469935352448, "mean": 0.00019329176166138495, "normalized": [ 1.8891850069388627, 1.7801603926271052, 1.6773544463608039, 1.566500262900107, 1.476456430986117, 1.3754925511314788, 1.3084504582544147, 1.236424592515421, 1.172965600373877, 1.150871569148191, 1.0802294364746856, 1.0319209662753672, 1.0080063247818516, 0.9698985570693867, 0.9067931019504507, 0.8756163462587222, 0.8444435806568995, 0.8113939671049883, 0.7809677247391495, 0.7613442367276676, 0.7402582678385915, 0.7293562882295931, 0.7162426695419845, 0.7062419232587185, 0.6940340549797547, 0.6866100033591334, 0.6831688143117768, 0.6753320518823956, 0.6715919825156708, 0.6686410475337363, 0.6640239865195754, 0.6600233567535221 ], "peak_layer": 1, "peak_normalized": 1.8891850069388627, "population_cv": 0.357915029863326, "rest_mean": 0.00020341356920888993, "spike_contrast": 0.6815376252628051, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001386340009048581, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00036516389809548855, 0.0003440903383307159, 0.0003242187958676368, 0.0003027915954589844, 0.00028538686456158757, 0.00026587137836031616, 0.0002529126941226423, 0.00023899068764876574, 0.00022672458726447076, 0.00022245399304665625, 0.0002087994507746771, 0.00019946182146668434, 0.00019483931828290224, 0.00018747340072877705, 0.00017527563613839447, 0.00016924942610785365, 0.00016322398732881993, 0.00015683576930314302, 0.0001509546273155138, 0.00014716156874783337, 0.0001430858246749267, 0.0001409785618307069, 0.00013844380737282336, 0.00013651074550580233, 0.00013415106514003128, 0.00013271605712361634, 0.00013205090363044292, 0.00013053612201474607, 0.00012981319741811603, 0.00012924280599690974, 0.00012835036613978446, 0.00012757707736454904 ] } } }, "pre_attention_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "de71d4cf2d509a1063f6eac7f50c4885e9a058dee7e4b993ad40f5527f528d00", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.02802751213312149, 0.01413465291261673, 0.014225807972252369, 0.01433455292135477, 0.01440478302538395, 0.009703479707241058, 0.00977881159633398, 0.009858808480203152, 0.009950309991836548, 0.00752287870272994, 0.007571155205368996, 0.007693085819482803, 0.007788818795233965, 0.006353048607707024, 0.006389216054230928, 0.0064707002602517605, 0.00656962301582098, 0.005553483963012695, 0.0056249531917274, 0.005688921548426151, 0.005768724251538515, 0.004966245498508215, 0.005057469941675663, 0.005098348017781973, 0.005216081161051989, 0.0046758754178881645, 0.004727107938379049, 0.004777040798217058, 0.004837613552808762, 0.0044186655431985855, 0.004510571248829365, 0.0045849340967834 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.0001580166108396952, "first_to_last_ratio": 1.9713740353643698, "imbalance_abs_log_ratio": 0.6787307795042399, "last_quartile_mean": 8.015557068574708e-05, "mean": 0.00011843875881822896, "normalized": [ 0.8918679597819287, 1.5814819593588525, 1.5722239987443252, 1.3246067564285042, 1.1252650527356378, 1.512743589128775, 1.3788112428745087, 1.2863035124436846, 1.097095055843533, 1.468154353173571, 1.1523254606068964, 1.0839888571909224, 1.054518217028451, 1.146335080549274, 1.1665208527784754, 0.8931136212023528, 0.8203804970547371, 1.073741223277434, 0.919654742394771, 0.8223971330190447, 0.7711288002244273, 0.8470775332584999, 0.8298711629738247, 0.7662487463744663, 0.7210382462310316, 0.7071376059209309, 0.7189559392079661, 0.727980518763353, 0.6111916361855597, 0.6986829946379066, 0.6244678178312321, 0.6046898327751223 ], "peak_layer": 2, "peak_normalized": 1.5814819593588525, "population_cv": 0.2904715201464359, "rest_mean": 0.00012310891098523926, "spike_contrast": 0.7572151875143335, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.321993711637333e-05, "top_five_layers": [ 2, 3, 6, 10, 7 ], "top_five_spike_overlap": 0 }, "values": [ 0.00010563173418631777, 0.0001873087603598833, 0.00018621225899551064, 0.00015688478015363216, 0.0001332749961875379, 0.00017916747310664505, 0.00016330469225067645, 0.00015234819147735834, 0.00012993857671972364, 0.00017388637934345752, 0.0001364799973089248, 0.0001283862948184833, 0.00012489582877606153, 0.00013577050413005054, 0.00013816128193866462, 0.00010577926877886057, 9.71648478298448e-05, 0.00012717257777694613, 0.00010892276623053476, 9.74036956904456e-05, 9.133153798757121e-05, 0.0001003268116619438, 9.828891052166e-05, 9.07535504666157e-05, 8.539887494407594e-05, 8.375250035896897e-05, 8.515224908478558e-05, 8.622110908618197e-05, 7.238877878990024e-05, 8.275114669231698e-05, 7.396119326585904e-05, 7.161871326388791e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.0001664285200604354, "first_to_last_ratio": 1.9719337364651075, "imbalance_abs_log_ratio": 0.6790146534176558, "last_quartile_mean": 8.439863722742302e-05, "mean": 0.0001245867656507471, "normalized": [ 0.8984334461398115, 1.5927952483854981, 1.5658595485525297, 1.3198735865176374, 1.131213894321221, 1.5125315092450449, 1.3796810696180641, 1.286366052942558, 1.0960606282761096, 1.4536884557404346, 1.1543342746982923, 1.0797559630746076, 1.0497530570909623, 1.1457182987306291, 1.1551962694199982, 0.9023721663305979, 0.8238710779144349, 1.0632503993742801, 0.9211624832492573, 0.8274335806257342, 0.7739032412019898, 0.850990856837025, 0.8281317613098801, 0.7681943949939335, 0.7208251570035091, 0.7089546837555952, 0.7203162530736348, 0.7236174125342815, 0.6145740290254166, 0.6994991973458914, 0.6274381906330746, 0.6042038120380651 ], "peak_layer": 2, "peak_normalized": 1.5927952483854981, "population_cv": 0.2889665774838781, "rest_mean": 0.00012946851159169995, "spike_contrast": 0.7586812913967177, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.822533756960183e-05, "top_five_layers": [ 2, 3, 6, 10, 7 ], "top_five_spike_overlap": 0 }, "values": [ 0.00011193291720701382, 0.0001984412083402276, 0.0001950853766174987, 0.000164438781212084, 0.00014093428035266697, 0.00018844140868168324, 0.00017189000209327787, 0.00016026418597903103, 0.00013655464863404632, 0.00018111034296453, 0.00014381477376446128, 0.0001345233031315729, 0.00013078533811494708, 0.00014274133718572557, 0.00014392216689884663, 0.00011242362961638719, 0.00010264343291055411, 0.0001324669283349067, 0.00011476465442683548, 0.00010308727360097691, 9.641810174798593e-05, 0.00010602219845168293, 0.00010317425767425448, 9.570685506332666e-05, 8.980527491075918e-05, 8.832637104205787e-05, 8.974187221610919e-05, 9.015315299620852e-05, 7.656779052922502e-05, 8.714834257261828e-05, 7.817049481673166e-05, 7.527579873567447e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.14759047608822584, "first_to_last_ratio": 1.9719337947467823, "imbalance_abs_log_ratio": 0.6790146829732503, "last_quartile_mean": 0.07484555337578058, "mean": 0.1104847895912826, "normalized": [ 0.8984334974418968, 1.5927952138130326, 1.565859646322909, 1.3198735575362357, 1.1312139236939245, 1.5125315007911495, 1.3796810264894532, 1.286366010451999, 1.0960606168617568, 1.4536884922319049, 1.1543342013148132, 1.0797558953237962, 1.0497530983161607, 1.1457182562234747, 1.1551962948455465, 0.9023721940039697, 0.8238711061937815, 1.0632504861804335, 0.9211625151546727, 0.8274335809310764, 0.7739032642161865, 0.8509909070635955, 0.8281317391228538, 0.7681943896830757, 0.7208251689669645, 0.7089546589522648, 0.7203162343708721, 0.7236173971219663, 0.6145740294939214, 0.6994991437375198, 0.6274381319743458, 0.6042038211744466 ], "peak_layer": 2, "peak_normalized": 1.5927952138130326, "population_cv": 0.28896658018674404, "rest_mean": 0.11481397047086998, "spike_contrast": 0.7586813040631772, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.08710721284151077, "top_five_layers": [ 2, 3, 6, 10, 7 ], "top_five_spike_overlap": 0 }, "values": [ 0.09926323592662811, 0.1759796440601349, 0.1730036735534668, 0.14582595229148865, 0.12498193234205246, 0.16711172461509705, 0.15243376791477203, 0.14212387800216675, 0.12109802663326263, 0.16061046719551086, 0.1275363713502884, 0.1192966029047966, 0.11598175019025803, 0.12658444046974182, 0.12763161957263947, 0.09969840198755264, 0.0910252258181572, 0.11747300624847412, 0.10177444666624069, 0.09141882508993149, 0.08550453931093216, 0.09402155131101608, 0.09149596095085144, 0.08487379550933838, 0.0796402171254158, 0.0783287063241005, 0.07958398759365082, 0.07994871586561203, 0.06790108233690262, 0.07728401571512222, 0.069322369992733, 0.06675533205270767 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 5.093040158499207e-05, "first_to_last_ratio": 0.8860093039525244, "imbalance_abs_log_ratio": 0.12102782735710801, "last_quartile_mean": 5.748291960117058e-05, "mean": 5.7093861016710434e-05, "normalized": [ 0.4337136137178416, 0.9185569667519601, 0.9444095926066538, 0.8728787927927439, 0.7849140301431445, 1.1296292591692207, 1.0726593944235638, 0.9796135302182263, 0.9386642177229508, 1.3046353648934683, 1.0461832118506482, 1.0032917361864164, 0.9662905330123907, 1.0760080966276753, 1.1828739682986207, 0.9087955578115051, 0.9194941496879511, 1.198292373503509, 1.0946132296674007, 1.0673695485587213, 0.9244260207272295, 1.0525679458300632, 1.125653431968099, 0.9999504891362015, 0.9629027708994021, 1.0063234987710272, 0.9975933861195996, 1.0791525148711678, 0.9412468232151083, 1.085173475571988, 1.0031556955576197, 0.9789667796878816 ], "peak_layer": 10, "peak_normalized": 1.3046353648934683, "population_cv": 0.1441687540611157, "rest_mean": 5.695535414647397e-05, "spike_contrast": 1.0155638391297448, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.784179811598733e-05, "top_five_layers": [ 10, 18, 15, 6, 23 ], "top_five_spike_overlap": 1 }, "values": [ 2.4762384782661684e-05, 5.2443963795667514e-05, 5.3919990023132414e-05, 4.9836020480142906e-05, 4.481377254705876e-05, 6.449489592341706e-05, 6.124226638348773e-05, 5.592991874436848e-05, 5.3591964388033375e-05, 7.448667020071298e-05, 5.973063889541663e-05, 5.728179894504137e-05, 5.516925739357248e-05, 6.143345672171563e-05, 6.75348419463262e-05, 5.1886647270293906e-05, 5.249747118796222e-05, 6.84151382301934e-05, 6.249569560168311e-05, 6.094024865888059e-05, 5.277905074763112e-05, 6.009516800986603e-05, 6.426790059776977e-05, 5.7091034250333905e-05, 5.4975836974335834e-05, 5.74548939766828e-05, 5.695645813830197e-05, 6.161298369988799e-05, 5.373941530706361e-05, 6.19567435933277e-05, 5.727403186028823e-05, 5.58929932594765e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 5.801741053801379e-05, "first_to_last_ratio": 0.954195857595535, "imbalance_abs_log_ratio": 0.046886327142486896, "last_quartile_mean": 6.08024129178375e-05, "mean": 6.296396202287724e-05, "normalized": [ 0.4796364558374551, 0.9568353419130547, 0.9804638447804884, 0.8949545354119591, 0.8237377234766136, 1.1597846916175047, 1.085649359633547, 0.9904450076993521, 0.9595078978373608, 1.2941748206155441, 1.0680753760655664, 1.0231559828609242, 0.9904019047543832, 1.096490963646882, 1.1850351594596897, 0.9422215946487361, 0.9418631353047864, 1.1705024179981143, 1.0926353878857977, 1.0382275689133567, 0.9320833313178482, 1.0784583323883348, 1.0985080196909318, 0.9917906747078287, 0.9476388143731139, 0.9855066582268529, 0.978396290111904, 1.028407839812513, 0.8982055143685478, 1.0282804954549911, 0.9451124964564701, 0.9138123627295472 ], "peak_layer": 10, "peak_normalized": 1.2941748206155441, "population_cv": 0.13431233496292436, "rest_mean": 6.285090866084729e-05, "spike_contrast": 1.0115120295379672, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.357445017783903e-05, "top_five_layers": [ 10, 15, 18, 6, 23 ], "top_five_spike_overlap": 1 }, "values": [ 3.019981159013696e-05, 6.0246144130360335e-05, 6.173388828756288e-05, 5.634988337988034e-05, 5.186579073779285e-05, 7.302463927771896e-05, 6.835678505012766e-05, 6.236234185053036e-05, 6.041441884008236e-05, 8.148637425620109e-05, 6.725025741616264e-05, 6.442195444833487e-05, 6.235962791834027e-05, 6.903941539349034e-05, 7.461450877599418e-05, 5.9326004702597857e-05, 5.930343468207866e-05, 7.369946979451925e-05, 6.879665306769311e-05, 6.537092122016475e-05, 5.8687659475253895e-05, 6.790400948375463e-05, 6.916641723364592e-05, 6.244707037694752e-05, 5.966709431959316e-05, 6.205140380188823e-05, 6.160370685392991e-05, 6.475263216998428e-05, 5.655457789544016e-05, 6.474461406469345e-05, 5.950802733423188e-05, 5.753724690293893e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 1.7888660295284353e-05, "first_to_last_ratio": 0.41855462748958633, "imbalance_abs_log_ratio": 0.8709478659353102, "last_quartile_mean": 4.2739129185065394e-05, "mean": 3.1986895351110434e-05, "normalized": [ 0.2593955501951102, 0.4997829486650695, 0.48480994473028793, 0.4968760363401999, 0.5474125689219601, 0.7788406177150305, 0.7554473604850604, 0.6514322393717875, 0.7578135841604356, 0.9674810188795085, 0.9032772556875033, 0.9019220651946442, 0.8311356963419073, 0.9587970177975769, 1.1401394869948454, 0.8953886492354074, 1.0262272842074007, 1.1413916918814533, 1.174037958462901, 1.194927715786966, 1.0267140632187253, 1.3434898267078448, 1.3729067467361273, 1.2011929483563188, 1.1735959904166378, 1.3266745689353452, 1.2739952988364465, 1.4049535244931144, 1.2752889026856997, 1.4680979746684197, 1.4182292846928506, 1.3483241791974143 ], "peak_layer": 30, "peak_normalized": 1.4680979746684197, "population_cv": 0.3184955307507873, "rest_mean": 3.066251991657299e-05, "spike_contrast": 1.276428773763581, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.913852269761264e-05, "top_five_layers": [ 30, 31, 28, 23, 32 ], "top_five_spike_overlap": 1 }, "values": [ 8.297258318634704e-06, 1.5986504877218977e-05, 1.5507564967265353e-05, 1.589352177688852e-05, 1.7510028555989265e-05, 2.491269333404489e-05, 2.416441566310823e-05, 2.083729486912489e-05, 2.4240103812189773e-05, 3.094671410508454e-05, 2.889303505071439e-05, 2.8849686714238487e-05, 2.6585450541460887e-05, 3.066893987124786e-05, 3.6469522456172854e-05, 2.8640703021665104e-05, 3.282582474639639e-05, 3.6509576602838933e-05, 3.755382931558415e-05, 3.822202779701911e-05, 3.2841395295690745e-05, 4.2974068492185324e-05, 4.391502443468198e-05, 3.842243313556537e-05, 3.7539692129939795e-05, 4.243620060151443e-05, 4.0751154301688075e-05, 4.494010136113502e-05, 4.0792532672639936e-05, 4.695989628089592e-05, 4.5364751713350415e-05, 4.3128704419359565e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00022091567916504573, "first_to_last_ratio": 1.232986382535498, "imbalance_abs_log_ratio": 0.20943917994887448, "last_quartile_mean": 0.00017917122386279516, "mean": 0.0002198539991695725, "normalized": [ 0.4672441326905094, 1.0299890566632133, 1.08986441571188, 1.0084939820045402, 0.8814237539069011, 1.2693826047962031, 1.1886514905905403, 1.1035827467935222, 1.0473307057033197, 1.4274044616309502, 1.1442998336482333, 1.0911283607583675, 1.1094038041384326, 1.1946873974128753, 1.2511466100570947, 0.9923165311956169, 0.9239703707694655, 1.2473578196113966, 1.1181336030529359, 1.0160745414007064, 0.9397394354798955, 0.9905446518023159, 1.0121913654482901, 0.9359943959598221, 0.8839446941554127, 0.8510229499852975, 0.8648513533654478, 0.8786898836618169, 0.7455559493639876, 0.8399702482950041, 0.7392957942269532, 0.7163130557190521 ], "peak_layer": 10, "peak_normalized": 1.4274044616309502, "population_cv": 0.18952709251819305, "rest_mean": 0.0002217885959436427, "spike_contrast": 0.9441746799407325, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00020940717658959329, "top_five_layers": [ 10, 6, 15, 18, 14 ], "top_five_spike_overlap": 0 }, "values": [ 0.00010272549116052687, 0.00022644721320830286, 0.00023961105034686625, 0.00022172143508214504, 0.0001937845372594893, 0.0002790788421407342, 0.0002613297838252038, 0.00024262708029709756, 0.00023025984410196543, 0.000313820579322055, 0.00025157889467664063, 0.00023988893372006714, 0.00024390686303377151, 0.000262656802078709, 0.00027506958576850593, 0.0002181647578254342, 0.00020313858112785965, 0.00027423660503700376, 0.0002458261442370713, 0.00022338805138133466, 0.00020660547306761146, 0.00021777520305477083, 0.00022253431961871684, 0.00020578211115207523, 0.0001943387760547921, 0.00018710079893935472, 0.0001901410287246108, 0.00019318348495289683, 0.00016391345707233995, 0.00018467081827111542, 0.00016253713693004102, 0.00015748428995721042 ] } } }, "pre_mlp_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "2999a257265024350fd2665e2a8d452a03cd662193b4245d15290408ad29922d", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.014061377383768559, 0.014165290631353855, 0.014281988143920898, 0.014350161887705326, 0.00966146495193243, 0.009741888381540775, 0.009820535778999329, 0.00991273857653141, 0.0074927168898284435, 0.007546746172010899, 0.00766373984515667, 0.007762390188872814, 0.006325984839349985, 0.006375366821885109, 0.006445344537496567, 0.006555322092026472, 0.005531130358576775, 0.005611324217170477, 0.0056777093559503555, 0.00574630219489336, 0.004946626722812653, 0.0050392793491482735, 0.005084683652967215, 0.005203457549214363, 0.004667261149734259, 0.004717131145298481, 0.004768412560224533, 0.004826710093766451, 0.0044052014127373695, 0.004504534415900707, 0.004574235528707504, 0.004644096828997135 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 3.729895206561196e-05, "first_to_last_ratio": 1.2469721805352876, "imbalance_abs_log_ratio": 0.22071835733540474, "last_quartile_mean": 2.9911615229138988e-05, "mean": 3.340667609563752e-05, "normalized": [ 1.308752778077389, 1.2456464596610868, 1.0236956931411572, 0.9922285647386262, 1.26903181062375, 1.1170846373362797, 1.0287316533628064, 0.9469237624012169, 1.1300261751959062, 1.1132698789554223, 1.0126240729566007, 0.9379202574439188, 1.1130503370514597, 1.070713488608195, 0.9897642722064988, 0.922085688720865, 1.036831181462568, 0.9898105546019274, 0.9395130252451869, 0.9001967292772246, 0.9975952535130206, 0.9404117749145336, 0.9224862220158919, 0.8885787590271026, 0.9461670180110204, 0.8974372095121397, 0.8904691498559152, 0.8684437934704528, 0.9244147277585869, 0.9015689206268677, 0.8740901912628669, 0.8604359589635168 ], "peak_layer": 1, "peak_normalized": 1.308752778077389, "population_cv": 0.1164905978879517, "rest_mean": 3.378375206213807e-05, "spike_contrast": 0.9285666618330386, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.137046587653458e-05, "top_five_layers": [ 1, 5, 2, 9, 6 ], "top_five_spike_overlap": 0 }, "values": [ 4.3721080146497115e-05, 4.161290780757554e-05, 3.419827044126578e-05, 3.314705827506259e-05, 4.239413465256803e-05, 3.73180846509058e-05, 3.4366505133220926e-05, 3.1633575417799875e-05, 3.7750418414361775e-05, 3.719064625329338e-05, 3.3828404411906376e-05, 3.1332798243965954e-05, 3.718331208801828e-05, 3.5768978705164045e-05, 3.306473445263691e-05, 3.080381793552078e-05, 3.463708344497718e-05, 3.3066280593629926e-05, 3.138600732199848e-05, 3.007258055731654e-05, 3.332634150865488e-05, 3.1416031561093405e-05, 3.0817198421573266e-05, 2.9684462788281962e-05, 3.1608295103069395e-05, 2.9980394174344838e-05, 2.9747614462394267e-05, 2.9011820515734144e-05, 3.0881623388268054e-05, 3.01184209092753e-05, 2.9200447897892445e-05, 2.874430538213346e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 6.705871965095866e-05, "first_to_last_ratio": 0.7180297289018083, "imbalance_abs_log_ratio": 0.33124430563536084, "last_quartile_mean": 9.33926785364747e-05, "mean": 8.228966771639534e-05, "normalized": [ 0.8685826408392191, 0.8249006353131848, 0.7154905339803773, 0.6884730939760333, 0.9470487095509537, 0.8638069620260335, 0.8256155016985178, 0.7853664463613442, 0.9851686390305121, 0.9692511243547932, 0.9083804947955202, 0.8729855457896263, 1.0422548534804916, 1.0255492616133721, 0.9871528463964067, 0.9398223255303129, 1.0958375595009802, 1.0726325633807163, 1.0495259774381736, 1.0230838788771643, 1.1576441025653552, 1.1142532603054807, 1.1008567438213033, 1.056908774400763, 1.159063578749664, 1.1192434432436902, 1.1032241584740294, 1.1060435702772593, 1.1786235078649367, 1.1628365878275773, 1.141840825041133, 1.1085318534950752 ], "peak_layer": 29, "peak_normalized": 1.1786235078649367, "population_cv": 0.1341844256762087, "rest_mean": 8.049536754023629e-05, "spike_contrast": 1.1426606459269575, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.197888866765424e-05, "top_five_layers": [ 29, 30, 25, 21, 31 ], "top_five_spike_overlap": 2 }, "values": [ 7.14753768988885e-05, 6.788079917896539e-05, 5.887747829547152e-05, 5.66542221349664e-05, 7.793232362018898e-05, 7.108238787623122e-05, 6.793962529627606e-05, 6.462754390668124e-05, 8.106919995043427e-05, 7.975935295689851e-05, 7.475032907677814e-05, 7.183769048424438e-05, 8.576680556870997e-05, 8.4392107964959e-05, 8.123247971525416e-05, 7.733766688033938e-05, 9.017610864248127e-05, 8.826657722238451e-05, 8.636514394311234e-05, 8.418923243880272e-05, 9.526214853394777e-05, 9.169153054244816e-05, 9.0589135652408e-05, 8.697267185198143e-05, 9.537895675748587e-05, 9.210217103827745e-05, 9.078394941752777e-05, 9.101595787797123e-05, 9.698853682493791e-05, 9.56894364207983e-05, 9.396170207764953e-05, 9.122071787714958e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.05946834245696664, "first_to_last_ratio": 0.7180297222365345, "imbalance_abs_log_ratio": 0.33124431491808765, "last_quartile_mean": 0.08282156102359295, "mean": 0.07297530013602227, "normalized": [ 0.8685826122359729, 0.8249005917343738, 0.71549052911611, 0.6884731350635278, 0.9470486699630107, 0.8638070113577097, 0.8256154769796886, 0.7853664269052657, 0.9851686341386658, 0.9692511556795396, 0.9083805454214465, 0.8729855579949275, 1.0422548040385637, 1.0255492365750694, 0.9871528950273681, 0.9398223256645052, 1.0958375053304956, 1.07263252629438, 1.0495259690486791, 1.0230837911446407, 1.1576441435704576, 1.1142533045483842, 1.1008568124470575, 1.0569088284972943, 1.1590636022344982, 1.119243411839159, 1.1032241423789504, 1.1060435591240758, 1.1786235037173105, 1.1628366076958818, 1.141840809884862, 1.1085318743481278 ], "peak_layer": 29, "peak_normalized": 1.1786235037173105, "population_cv": 0.13418442668798974, "rest_mean": 0.07138409617322462, "spike_contrast": 1.1426607032635474, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.08156780153512955, "top_five_layers": [ 29, 30, 25, 21, 31 ], "top_five_spike_overlap": 2 }, "values": [ 0.06338507682085037, 0.0601973682641983, 0.05221313610672951, 0.050241533666849136, 0.0691111609339714, 0.06303657591342926, 0.060249537229537964, 0.05731235072016716, 0.07189297676086426, 0.07073139399290085, 0.06628934293985367, 0.06370638310909271, 0.07605885714292526, 0.0748397633433342, 0.07203777879476547, 0.06858381628990173, 0.07996907085180283, 0.07827568054199219, 0.07658947259187698, 0.07465984672307968, 0.0844794288277626, 0.08131296932697296, 0.0803353562951088, 0.07712823897600174, 0.08458301424980164, 0.08167712390422821, 0.08050811290740967, 0.08071386069059372, 0.0860104039311409, 0.08485835045576096, 0.08332617580890656, 0.08089544624090195 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 7.408341275549901e-06, "first_to_last_ratio": 0.49124437919492936, "imbalance_abs_log_ratio": 0.7108135577065074, "last_quartile_mean": 1.5080765479069669e-05, "mean": 1.1844782648040564e-05, "normalized": [ 0.5657990507718705, 0.5382233871697658, 0.49942145823009426, 0.48906984415931193, 0.8062921756393574, 0.7122953827323945, 0.7013508394120195, 0.6911626993599347, 0.8940675881372303, 0.868722127857376, 0.9735911331661544, 0.854855860776803, 0.989483587379368, 1.0059088486455374, 1.0303729006301217, 0.9941025538400318, 1.1620912657632405, 1.127012997009919, 1.113401270263767, 1.066097770938846, 1.1826034542812638, 1.1643547935214302, 1.2049778184223279, 1.1791491538213539, 1.2365172470752577, 1.219921447624247, 1.2337761202582145, 1.2288278254433027, 1.3039217543779833, 1.3631114810840155, 1.2856635723507386, 1.3138525898567213 ], "peak_layer": 30, "peak_normalized": 1.3631114810840155, "population_cv": 0.2549101540724733, "rest_mean": 1.1420299651271543e-05, "spike_contrast": 1.237882653019464, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.4136990830593277e-05, "top_five_layers": [ 30, 32, 29, 31, 25 ], "top_five_spike_overlap": 1 }, "values": [ 6.701766778860474e-06, 6.3751390371180605e-06, 5.915538622502936e-06, 5.7929260037781205e-06, 9.550355571263935e-06, 8.436983989668079e-06, 8.307348252856173e-06, 8.186671948351432e-06, 1.0590036254143342e-05, 1.0289824786013924e-05, 1.1531975360412616e-05, 1.0125581866304856e-05, 1.1720218026312068e-05, 1.1914771675947122e-05, 1.220454305439489e-05, 1.1774928680097219e-05, 1.3764718460151926e-05, 1.334922399109928e-05, 1.3187996046326589e-05, 1.2627696378331166e-05, 1.4007680874783546e-05, 1.379152945446549e-05, 1.4272700354922563e-05, 1.3966765436634887e-05, 1.4646278032159898e-05, 1.4449704394792207e-05, 1.4613809980801307e-05, 1.4555198504240252e-05, 1.5444669770658948e-05, 1.614575921848882e-05, 1.5228405572997872e-05, 1.5562298358418047e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 5.6029601637419546e-05, "first_to_last_ratio": 0.6070213107493427, "imbalance_abs_log_ratio": 0.49919138022039955, "last_quartile_mean": 9.230252817360451e-05, "mean": 7.840498903988191e-05, "normalized": [ 0.6598133090619259, 0.6419677920829977, 0.6005256373772865, 0.5913463573822231, 0.8562376513989928, 0.8062968559250673, 0.7923156566726156, 0.768439227921366, 0.9790122729268586, 0.9665730330981341, 0.9163710065350301, 0.8881686280668529, 1.0631190324760253, 1.0511030519715439, 1.0188080221730096, 0.9712812246823179, 1.1361272291219966, 1.1138561419734825, 1.0918134348286874, 1.0639217496772315, 1.203414696914147, 1.1592426077267617, 1.1442664103435203, 1.097952968622282, 1.2042281788778202, 1.1625445130141736, 1.1452986211445626, 1.1473932954411665, 1.2233942816523278, 1.205318760675343, 1.1829469858778916, 1.146901364356358 ], "peak_layer": 29, "peak_normalized": 1.2233942816523278, "population_cv": 0.1898597249088972, "rest_mean": 7.605543874820728e-05, "spike_contrast": 1.1977126437531995, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.109256061492487e-05, "top_five_layers": [ 29, 30, 25, 21, 31 ], "top_five_spike_overlap": 2 }, "values": [ 5.173265526536852e-05, 5.033347770222463e-05, 4.708420601673424e-05, 4.636450466932729e-05, 6.713330367347226e-05, 6.321769615169615e-05, 6.212150037754327e-05, 6.024946924299002e-05, 7.675944652874023e-05, 7.578414806630462e-05, 7.184805872384459e-05, 6.963685154914856e-05, 8.335383608937263e-05, 8.241172326961532e-05, 7.987963181221858e-05, 7.615329377586022e-05, 8.907804294722155e-05, 8.733187860343605e-05, 8.560362039133906e-05, 8.341677312273532e-05, 9.435371612198651e-05, 9.089040395338088e-05, 8.971619536168873e-05, 8.608499047113582e-05, 9.441749716643244e-05, 9.114928980125114e-05, 8.97971258382313e-05, 8.996135875349864e-05, 9.592021524440497e-05, 9.450300422031432e-05, 9.274894546251744e-05, 8.992278890218586e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 4.8511469685763586e-05, "first_to_last_ratio": 0.5271352332510719, "imbalance_abs_log_ratio": 0.6402981537748061, "last_quartile_mean": 9.202850924339145e-05, "mean": 7.532146378252946e-05, "normalized": [ 0.548104082267993, 0.5416830199005969, 0.528884702689083, 0.5247448675895723, 0.7777813723908718, 0.7487613756360456, 0.7496059866168319, 0.7329069161583202, 0.9480676391902623, 0.9399123881532756, 0.9000936195631334, 0.883441930780847, 1.0560589316963704, 1.0534552095740957, 1.032095781768693, 0.9835620691098637, 1.1601754560133901, 1.1467788545782247, 1.1233340534265408, 1.0967375255670042, 1.2396083970871608, 1.1951467072205837, 1.1828066060767042, 1.1317734013234224, 1.244698956870482, 1.2025862582796845, 1.1881970083126314, 1.1882256015362047, 1.2689460104605979, 1.2589162611011824, 1.2307315535498626, 1.1921774555104672 ], "peak_layer": 29, "peak_normalized": 1.2689460104605979, "population_cv": 0.23486334953575583, "rest_mean": 7.254842300123225e-05, "spike_contrast": 1.244629176846487, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.029588400153444e-05, "top_five_layers": [ 29, 30, 25, 21, 31 ], "top_five_spike_overlap": 2 }, "values": [ 4.1284001781605184e-05, 4.080035796505399e-05, 3.9836369978729635e-05, 3.952455153921619e-05, 5.858363147126511e-05, 5.639780283672735e-05, 5.6461420172126964e-05, 5.520362174138427e-05, 7.140984234865755e-05, 7.079557690303773e-05, 6.77963689668104e-05, 6.654213939327747e-05, 7.954390457598493e-05, 7.934778841445222e-05, 7.773896504659206e-05, 7.408333476632833e-05, 8.738611359149218e-05, 8.637706196168438e-05, 8.461116522084922e-05, 8.26078758109361e-05, 9.336911898571998e-05, 9.002019942272455e-05, 8.909072494134307e-05, 8.524682925781235e-05, 9.375254740007222e-05, 9.058055729838088e-05, 8.949673792812973e-05, 8.949889161158353e-05, 9.557887096889317e-05, 9.482341556577012e-05, 9.270050213672221e-05, 8.979655103757977e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 9.61669284151867e-05, "first_to_last_ratio": 0.8297833624837898, "imbalance_abs_log_ratio": 0.18659062131596024, "last_quartile_mean": 0.00011589401856326731, "mean": 0.00010716335873439675, "normalized": [ 0.9380742209455786, 0.8898820858744699, 0.7837417114644826, 0.7583102664636657, 1.0822070619439073, 0.9520749081713696, 0.9107982467806989, 0.8640017807748708, 1.0669538303069324, 1.0266950649381863, 0.9594029855154438, 0.8901506143407661, 1.0795289063552718, 1.0379165674077444, 0.9706971366471856, 0.914979822300863, 1.052939971267624, 1.0225707935964177, 0.9851238030379684, 0.9636652893153618, 1.0990044510799908, 1.0523904893769196, 1.0422234788441622, 1.0049019599938585, 1.0962247910521359, 1.0599378706264646, 1.05039930496957, 1.0583562345651827, 1.1199788667078585, 1.1100529546792268, 1.0850973923636869, 1.0717171382921347 ], "peak_layer": 29, "peak_normalized": 1.1199788667078585, "population_cv": 0.09156717959150794, "rest_mean": 0.00010599351123716958, "spike_contrast": 1.0706366257223145, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00011348053521942347, "top_five_layers": [ 29, 30, 21, 25, 31 ], "top_five_spike_overlap": 2 }, "values": [ 0.00010052718425868079, 9.536275319987908e-05, 8.398839418077841e-05, 8.12630751170218e-05, 0.00011597294360399246, 0.00010202754492638633, 9.760419925441965e-05, 9.25893327803351e-05, 0.00011433835607022047, 0.00011002409155480564, 0.00010281284630764276, 9.539152961224318e-05, 0.000115685943455901, 0.0001112266254494898, 0.0001040231654769741, 9.805231093196198e-05, 0.00011283658386673778, 0.00010958212078548968, 0.00010556917550275102, 0.00010326960909878835, 0.00011777300824178383, 0.0001127776995417662, 0.00011168816854478791, 0.00010768866923172027, 0.00011747513053705916, 0.00011358650226611644, 0.00011256431753281504, 0.00011341700883349404, 0.00012002069706795737, 0.00011895700299646705, 0.00011628268111962825, 0.00011484880815260112 ] } } } }, "selector": { "expected_uniform_count": 65, "identities_unique": true, "identity_order_sha256": "97fd9721a263024f6a746638f32e0ad9a988cd4c111c00e9e178c1022e5c9d5e", "passed": true, "uniform_count": 65, "uniform_indices": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64 ], "visit_count": 65, "visits": [ { "branch": "attention", "coefficient": "uniform", "group": 1, "index": 0, "kind": "depth", "layer": 1, "offset": 1, "sources": 1, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 1, "index": 1, "kind": "depth", "layer": 1, "offset": 1, "sources": 2, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 1, "index": 2, "kind": "depth", "layer": 2, "offset": 2, "sources": 2, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 1, "index": 3, "kind": "depth", "layer": 2, "offset": 2, "sources": 2, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 1, "index": 4, "kind": "depth", "layer": 3, "offset": 3, "sources": 2, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 1, "index": 5, "kind": "depth", "layer": 3, "offset": 3, "sources": 2, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 1, "index": 6, "kind": "depth", "layer": 4, "offset": 4, "sources": 2, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 1, "index": 7, "kind": "depth", "layer": 4, "offset": 4, "sources": 2, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 2, "index": 8, "kind": "depth", "layer": 5, "offset": 1, "sources": 2, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 2, "index": 9, "kind": "depth", "layer": 5, "offset": 1, "sources": 3, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 2, "index": 10, "kind": "depth", "layer": 6, "offset": 2, "sources": 3, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 2, "index": 11, "kind": "depth", "layer": 6, "offset": 2, "sources": 3, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 2, "index": 12, "kind": "depth", "layer": 7, "offset": 3, "sources": 3, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 2, "index": 13, "kind": "depth", "layer": 7, "offset": 3, "sources": 3, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 2, "index": 14, "kind": "depth", "layer": 8, "offset": 4, "sources": 3, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 2, "index": 15, "kind": "depth", "layer": 8, "offset": 4, "sources": 3, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 3, "index": 16, "kind": "depth", "layer": 9, "offset": 1, "sources": 3, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 3, "index": 17, "kind": "depth", "layer": 9, "offset": 1, "sources": 4, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 3, "index": 18, "kind": "depth", "layer": 10, "offset": 2, "sources": 4, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 3, "index": 19, "kind": "depth", "layer": 10, "offset": 2, "sources": 4, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 3, "index": 20, "kind": "depth", "layer": 11, "offset": 3, "sources": 4, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 3, "index": 21, "kind": "depth", "layer": 11, "offset": 3, "sources": 4, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 3, "index": 22, "kind": "depth", "layer": 12, "offset": 4, "sources": 4, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 3, "index": 23, "kind": "depth", "layer": 12, "offset": 4, "sources": 4, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 4, "index": 24, "kind": "depth", "layer": 13, "offset": 1, "sources": 4, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 4, "index": 25, "kind": "depth", "layer": 13, "offset": 1, "sources": 5, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 4, "index": 26, "kind": "depth", "layer": 14, "offset": 2, "sources": 5, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 4, "index": 27, "kind": "depth", "layer": 14, "offset": 2, "sources": 5, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 4, "index": 28, "kind": "depth", "layer": 15, "offset": 3, "sources": 5, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 4, "index": 29, "kind": "depth", "layer": 15, "offset": 3, "sources": 5, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 4, "index": 30, "kind": "depth", "layer": 16, "offset": 4, "sources": 5, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 4, "index": 31, "kind": "depth", "layer": 16, "offset": 4, "sources": 5, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 5, "index": 32, "kind": "depth", "layer": 17, "offset": 1, "sources": 5, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 5, "index": 33, "kind": "depth", "layer": 17, "offset": 1, "sources": 6, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 5, "index": 34, "kind": "depth", "layer": 18, "offset": 2, "sources": 6, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 5, "index": 35, "kind": "depth", "layer": 18, "offset": 2, "sources": 6, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 5, "index": 36, "kind": "depth", "layer": 19, "offset": 3, "sources": 6, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 5, "index": 37, "kind": "depth", "layer": 19, "offset": 3, "sources": 6, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 5, "index": 38, "kind": "depth", "layer": 20, "offset": 4, "sources": 6, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 5, "index": 39, "kind": "depth", "layer": 20, "offset": 4, "sources": 6, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 6, "index": 40, "kind": "depth", "layer": 21, "offset": 1, "sources": 6, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 6, "index": 41, "kind": "depth", "layer": 21, "offset": 1, "sources": 7, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 6, "index": 42, "kind": "depth", "layer": 22, "offset": 2, "sources": 7, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 6, "index": 43, "kind": "depth", "layer": 22, "offset": 2, "sources": 7, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 6, "index": 44, "kind": "depth", "layer": 23, "offset": 3, "sources": 7, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 6, "index": 45, "kind": "depth", "layer": 23, "offset": 3, "sources": 7, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 6, "index": 46, "kind": "depth", "layer": 24, "offset": 4, "sources": 7, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 6, "index": 47, "kind": "depth", "layer": 24, "offset": 4, "sources": 7, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 7, "index": 48, "kind": "depth", "layer": 25, "offset": 1, "sources": 7, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 7, "index": 49, "kind": "depth", "layer": 25, "offset": 1, "sources": 8, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 7, "index": 50, "kind": "depth", "layer": 26, "offset": 2, "sources": 8, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 7, "index": 51, "kind": "depth", "layer": 26, "offset": 2, "sources": 8, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 7, "index": 52, "kind": "depth", "layer": 27, "offset": 3, "sources": 8, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 7, "index": 53, "kind": "depth", "layer": 27, "offset": 3, "sources": 8, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 7, "index": 54, "kind": "depth", "layer": 28, "offset": 4, "sources": 8, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 7, "index": 55, "kind": "depth", "layer": 28, "offset": 4, "sources": 8, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 8, "index": 56, "kind": "depth", "layer": 29, "offset": 1, "sources": 8, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 8, "index": 57, "kind": "depth", "layer": 29, "offset": 1, "sources": 9, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 8, "index": 58, "kind": "depth", "layer": 30, "offset": 2, "sources": 9, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 8, "index": 59, "kind": "depth", "layer": 30, "offset": 2, "sources": 9, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 8, "index": 60, "kind": "depth", "layer": 31, "offset": 3, "sources": 9, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 8, "index": 61, "kind": "depth", "layer": 31, "offset": 3, "sources": 9, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 8, "index": 62, "kind": "depth", "layer": 32, "offset": 4, "sources": 9, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 8, "index": 63, "kind": "depth", "layer": 32, "offset": 4, "sources": 9, "use_uniform": true }, { "branch": "output", "coefficient": "uniform", "group": null, "index": 64, "kind": "output", "layer": null, "offset": null, "sources": 9, "use_uniform": true } ], "visits_sha256": "e8eb3dec5a2d1247a9dc230971db785b08d1d64920a8e4a5ecffd27a07d55f4c" } }, "uniform_depth_all": { "bits_per_byte": 7.959341947457508, "forward": { "activation_sha256": { "attention_branch_output": "676759a2d7c0615ad7e1f246e90927fb829d00c8b6c3745032ae6b27ab7d902a", "mlp_branch_output": "afe29f3ef4d2992353f0bb59fe7c5c2c5050fca671a53184825a8d63cd83a2bc", "post_attention_state": "31853ffb07b24cc59e774bb20656840b4358b3080b42516d380f2dc0f46404c8", "post_mlp_state": "7fd9c911e93175896ad618842c120e77b3f017c17b7eec9c2bfb01fc6dbc8ac2", "pre_attention_input": "de71d4cf2d509a1063f6eac7f50c4885e9a058dee7e4b993ad40f5527f528d00", "pre_mlp_input": "2999a257265024350fd2665e2a8d452a03cd662193b4245d15290408ad29922d" }, "logits_sha256": "873a309a197bfd08fa523edcb440d2697903a34abc02dad1c7f5991509c3c151", "loss_nats": 5.516995429992676, "mixer_summary_sha256": "8450b24bdc777885a3357dc7b8080e7410acbe96943686433da76f379295133a" }, "loss_nats": 5.516995429992676, "loss_scale": 1.0, "mixers": [ { "branch": "attention", "entropy_mean": -0.0, "group": 1, "latest_source_mass": 1.0, "layer": 1, "max_source_mass": 1.0, "mixer_index": 0, "normalized_entropy": 1.0, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 1.0, "median": 1.0, "p05": 1.0, "p95": 1.0 } ], "sources": 1 }, { "branch": "mlp", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 1, "max_source_mass": 0.5, "mixer_index": 1, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 2, "max_source_mass": 0.5, "mixer_index": 2, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 2, "max_source_mass": 0.5, "mixer_index": 3, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 3, "max_source_mass": 0.5, "mixer_index": 4, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 3, "max_source_mass": 0.5, "mixer_index": 5, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 4, "max_source_mass": 0.5, "mixer_index": 6, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 4, "max_source_mass": 0.5, "mixer_index": 7, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6931471824645996, "group": 2, "latest_source_mass": 0.5, "layer": 5, "max_source_mass": 0.5, "mixer_index": 8, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "completed_group_1", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 5, "max_source_mass": 0.3333333432674408, "mixer_index": 9, "normalized_entropy": 1.0000000180537645, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 6, "max_source_mass": 0.3333333432674408, "mixer_index": 10, "normalized_entropy": 1.0000000180537645, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 6, "max_source_mass": 0.3333333432674408, "mixer_index": 11, "normalized_entropy": 1.0000000180537645, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 7, "max_source_mass": 0.3333333432674408, "mixer_index": 12, "normalized_entropy": 1.0000000180537645, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 7, "max_source_mass": 0.3333333432674408, "mixer_index": 13, "normalized_entropy": 1.0000000180537645, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 8, "max_source_mass": 0.3333333432674408, "mixer_index": 14, "normalized_entropy": 1.0000000180537645, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 8, "max_source_mass": 0.3333333432674408, "mixer_index": 15, "normalized_entropy": 1.0000000180537645, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0986123085021973, "group": 3, "latest_source_mass": 0.3333333432674408, "layer": 9, "max_source_mass": 0.3333333432674408, "mixer_index": 16, "normalized_entropy": 1.0000000180537645, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_2", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 9, "max_source_mass": 0.25, "mixer_index": 17, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 10, "max_source_mass": 0.25, "mixer_index": 18, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 10, "max_source_mass": 0.25, "mixer_index": 19, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 11, "max_source_mass": 0.25, "mixer_index": 20, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 11, "max_source_mass": 0.25, "mixer_index": 21, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 12, "max_source_mass": 0.25, "mixer_index": 22, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 12, "max_source_mass": 0.25, "mixer_index": 23, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3862943649291992, "group": 4, "latest_source_mass": 0.25, "layer": 13, "max_source_mass": 0.25, "mixer_index": 24, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_3", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 13, "max_source_mass": 0.20000000298023224, "mixer_index": 25, "normalized_entropy": 1.0000000186840277, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 14, "max_source_mass": 0.20000000298023224, "mixer_index": 26, "normalized_entropy": 1.0000000186840277, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 14, "max_source_mass": 0.20000000298023224, "mixer_index": 27, "normalized_entropy": 1.0000000186840277, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 15, "max_source_mass": 0.20000000298023224, "mixer_index": 28, "normalized_entropy": 1.0000000186840277, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 15, "max_source_mass": 0.20000000298023224, "mixer_index": 29, "normalized_entropy": 1.0000000186840277, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 16, "max_source_mass": 0.20000000298023224, "mixer_index": 30, "normalized_entropy": 1.0000000186840277, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 16, "max_source_mass": 0.20000000298023224, "mixer_index": 31, "normalized_entropy": 1.0000000186840277, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.6094379425048828, "group": 5, "latest_source_mass": 0.20000000298023224, "layer": 17, "max_source_mass": 0.20000000298023224, "mixer_index": 32, "normalized_entropy": 1.0000000186840277, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_4", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 17, "max_source_mass": 0.1666666716337204, "mixer_index": 33, "normalized_entropy": 1.0000001451965654, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 18, "max_source_mass": 0.1666666716337204, "mixer_index": 34, "normalized_entropy": 1.0000001451965654, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 18, "max_source_mass": 0.1666666716337204, "mixer_index": 35, "normalized_entropy": 1.0000001451965654, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 19, "max_source_mass": 0.1666666716337204, "mixer_index": 36, "normalized_entropy": 1.0000001451965654, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 19, "max_source_mass": 0.1666666716337204, "mixer_index": 37, "normalized_entropy": 1.0000001451965654, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 20, "max_source_mass": 0.1666666716337204, "mixer_index": 38, "normalized_entropy": 1.0000001451965654, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 20, "max_source_mass": 0.1666666716337204, "mixer_index": 39, "normalized_entropy": 1.0000001451965654, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.791759729385376, "group": 6, "latest_source_mass": 0.1666666716337204, "layer": 21, "max_source_mass": 0.1666666716337204, "mixer_index": 40, "normalized_entropy": 1.0000001451965654, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_5", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 21, "max_source_mass": 0.1428571492433548, "mixer_index": 41, "normalized_entropy": 1.00000003408302, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 22, "max_source_mass": 0.1428571492433548, "mixer_index": 42, "normalized_entropy": 1.00000003408302, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 22, "max_source_mass": 0.1428571492433548, "mixer_index": 43, "normalized_entropy": 1.00000003408302, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 23, "max_source_mass": 0.1428571492433548, "mixer_index": 44, "normalized_entropy": 1.00000003408302, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 23, "max_source_mass": 0.1428571492433548, "mixer_index": 45, "normalized_entropy": 1.00000003408302, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 24, "max_source_mass": 0.1428571492433548, "mixer_index": 46, "normalized_entropy": 1.00000003408302, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 24, "max_source_mass": 0.1428571492433548, "mixer_index": 47, "normalized_entropy": 1.00000003408302, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9459102153778076, "group": 7, "latest_source_mass": 0.1428571492433548, "layer": 25, "max_source_mass": 0.1428571492433548, "mixer_index": 48, "normalized_entropy": 1.00000003408302, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_6", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 25, "max_source_mass": 0.125, "mixer_index": 49, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 26, "max_source_mass": 0.125, "mixer_index": 50, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 26, "max_source_mass": 0.125, "mixer_index": 51, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 27, "max_source_mass": 0.125, "mixer_index": 52, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 27, "max_source_mass": 0.125, "mixer_index": 53, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 28, "max_source_mass": 0.125, "mixer_index": 54, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 28, "max_source_mass": 0.125, "mixer_index": 55, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.079441547393799, "group": 8, "latest_source_mass": 0.125, "layer": 29, "max_source_mass": 0.125, "mixer_index": 56, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_7", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 29, "max_source_mass": 0.1111111119389534, "mixer_index": 57, "normalized_entropy": 1.0000000180537645, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 30, "max_source_mass": 0.1111111119389534, "mixer_index": 58, "normalized_entropy": 1.0000000180537645, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 30, "max_source_mass": 0.1111111119389534, "mixer_index": 59, "normalized_entropy": 1.0000000180537645, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 31, "max_source_mass": 0.1111111119389534, "mixer_index": 60, "normalized_entropy": 1.0000000180537645, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 31, "max_source_mass": 0.1111111119389534, "mixer_index": 61, "normalized_entropy": 1.0000000180537645, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 32, "max_source_mass": 0.1111111119389534, "mixer_index": 62, "normalized_entropy": 1.0000000180537645, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 32, "max_source_mass": 0.1111111119389534, "mixer_index": 63, "normalized_entropy": 1.0000000180537645, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "output", "entropy_mean": 2.1972246170043945, "group": null, "latest_source_mass": 0.1111111119389534, "layer": null, "max_source_mass": 0.1111111119389534, "mixer_index": null, "normalized_entropy": 1.0000000180537645, "offset": null, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_8", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 } ], "mode": "uniform_depth_all", "optimizer_state_after": "b82a349700eda58e2330a22ca2ee1f7fb1406efd47fd9aa306122b368a114192", "optimizer_state_before": "b82a349700eda58e2330a22ca2ee1f7fb1406efd47fd9aa306122b368a114192", "optimizer_state_unchanged": true, "positions": { "attention_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "676759a2d7c0615ad7e1f246e90927fb829d00c8b6c3745032ae6b27ab7d902a", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.0022265110164880753, 0.0021613112185150385, 0.0025601957459002733, 0.0025323182344436646, 0.0027253651060163975, 0.002904455177485943, 0.0029208383057266474, 0.00299624796025455, 0.0030937574338167906, 0.0033804215490818024, 0.0034370317589491606, 0.0038969130255281925, 0.004255065228790045, 0.004119740333408117, 0.003970506135374308, 0.004385598469525576, 0.0041999719105660915, 0.005232699681073427, 0.004664790350943804, 0.004898969549685717, 0.005131533369421959, 0.005069608334451914, 0.0051318020559847355, 0.0058372304774820805, 0.0059690922498703, 0.00554921617731452, 0.0058550527319312096, 0.006255472078919411, 0.006502739619463682, 0.006122888531535864, 0.007071305997669697, 0.006241494789719582 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.00015962477027642308, "first_to_last_ratio": 3.7699040072151497, "imbalance_abs_log_ratio": 1.3270495388586014, "last_quartile_mean": 4.234186599205714e-05, "mean": 8.617327489446325e-05, "normalized": [ 2.5428430303860874, 2.3222064178412967, 2.1011513502014654, 1.8911470095267005, 1.7004447966680725, 1.5353700031893112, 1.4229335846259188, 1.3028620492588128, 1.2049362956920768, 1.1705197150696323, 1.0773612761873523, 1.0133200843661747, 0.9906112963967714, 0.9272530070793641, 0.8550207003106389, 0.8185600560490035, 0.7703816559177782, 0.7173918665339755, 0.6760157258502337, 0.6514535263006792, 0.6184138115448249, 0.6053606047045522, 0.5838821473278404, 0.5697017460729616, 0.5457761369570343, 0.5276086337508958, 0.5104655265174319, 0.4951752820428687, 0.47527333481689216, 0.465789239220051, 0.45896414036514044, 0.45180594922816086 ], "peak_layer": 1, "peak_normalized": 2.5428430303860874, "population_cv": 0.5705341151299013, "rest_mean": 9.280180475562466e-05, "spike_contrast": 0.5428688997682254, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.0379213644191625e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00021912511147093028, 0.00020011213200632483, 0.00018106309289578348, 0.00016296633111778647, 0.00014653289690613747, 0.00013230786134954542, 0.0001226188469445333, 0.00011227188952034339, 0.0001038333066389896, 0.00010086751717608422, 9.283974941354245e-05, 8.732111018616706e-05, 8.536421955795959e-05, 7.990442827576771e-05, 7.367993384832516e-05, 7.053800072753802e-05, 6.63863102090545e-05, 6.182000652188435e-05, 5.825448897667229e-05, 5.6137883802875876e-05, 5.329074338078499e-05, 5.216590579948388e-05, 5.031503678765148e-05, 4.909306517220102e-05, 4.703131708083674e-05, 4.5465763832908124e-05, 4.3988486140733585e-05, 4.2670875700423494e-05, 4.095585973118432e-05, 4.013858415419236e-05, 3.955044303438626e-05, 3.893359826179221e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.00021442661636683624, "first_to_last_ratio": 1.843144421770586, "imbalance_abs_log_ratio": 0.6114730379455638, "last_quartile_mean": 0.0001163373926829081, "mean": 0.00015221274247778638, "normalized": [ 1.7741446897870732, 1.648461275666186, 1.5301464224684427, 1.4217121808823485, 1.3265065496562456, 1.2454420544341354, 1.1897998831173078, 1.1336248235930768, 1.0869089599614004, 1.0643003125599004, 1.0227030055526172, 0.9941765648180964, 0.9788958472869692, 0.9511102301475627, 0.9228060651723192, 0.9052418824754537, 0.8846484396889256, 0.865292475366091, 0.8497311645646503, 0.8388691935254817, 0.8256303537004397, 0.8176325372729673, 0.8076852907347736, 0.8000670639426607, 0.7892588692968658, 0.7814415503461742, 0.7731202151361662, 0.7664994567889658, 0.758490072461763, 0.7529625284402992, 0.7485174912671357, 0.7441725498875049 ], "peak_layer": 1, "peak_normalized": 1.7741446897870732, "population_cv": 0.27364872432853743, "rest_mean": 0.00015762320577373935, "spike_contrast": 0.7803181014868814, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012299624067964033, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00027004742878489196, 0.0002509168116375804, 0.0002329077833564952, 0.00021640271006617695, 0.00020191119983792305, 0.00018957215070258826, 0.0001811027032090351, 0.00017255214333999902, 0.00016544139361940324, 0.00016200006939470768, 0.00015566842921543866, 0.0001513263414381072, 0.00014900042151566595, 0.0001447710965294391, 0.00014046284195501357, 0.0001377893495373428, 0.00013465476513374597, 0.00013170854072086513, 0.00012933991092722863, 0.0001276865805266425, 0.00012567146040964872, 0.00012445409083738923, 0.00012293999316170812, 0.00012178040196886286, 0.00012013525702059269, 0.00011894536146428436, 0.00011767874821089208, 0.00011667098442558199, 0.00011545185407157987, 0.00011461049143690616, 0.00011393390013836324, 0.00011327254469506443 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.19015566632151604, "first_to_last_ratio": 1.8431444167877922, "imbalance_abs_log_ratio": 0.6114730352421437, "last_quartile_mean": 0.10316916275769472, "mean": 0.13498378219082952, "normalized": [ 1.7741447141792424, 1.6484611693507762, 1.5301464164964773, 1.421712099711509, 1.3265065258282553, 1.245442086656075, 1.1897999853143832, 1.1336247998774232, 1.0869090145314644, 1.0643003539468234, 1.0227030148964142, 0.9941765578539046, 0.978895844117059, 0.9511102293531465, 0.9228060470726628, 0.9052419251832852, 0.8846484748200404, 0.8652925213936268, 0.8497311905993837, 0.8388692019864461, 0.8256303029992552, 0.8176324960789627, 0.8076852727084025, 0.800067049482784, 0.789258877083647, 0.7814415614650243, 0.7731201949085273, 0.7664994759042474, 0.758490077799444, 0.7529625178867759, 0.7485174642151933, 0.7441725362993381 ], "peak_layer": 1, "peak_normalized": 1.7741447141792424, "population_cv": 0.2736487184871982, "rest_mean": 0.13978183573042904, "spike_contrast": 0.7803180757143806, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.10907429307699204, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.23948076367378235, 0.2225155234336853, 0.20654495060443878, 0.1919080764055252, 0.17905686795711517, 0.16811448335647583, 0.16060370206832886, 0.1530209630727768, 0.14671508967876434, 0.14366328716278076, 0.13804832100868225, 0.13419771194458008, 0.1321350634098053, 0.12838445603847504, 0.12456385046243668, 0.12219297885894775, 0.11941319704055786, 0.116800457239151, 0.11469992995262146, 0.11323373764753342, 0.11144670099020004, 0.11036712676286697, 0.10902441293001175, 0.10799607634544373, 0.10653714835643768, 0.10548193752765656, 0.10435868799686432, 0.1034649983048439, 0.10238385945558548, 0.10163772851228714, 0.10103771835565567, 0.10045122355222702 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 6.883365585963475e-05, "first_to_last_ratio": 2.2497546817252747, "imbalance_abs_log_ratio": 0.8108211799277664, "last_quartile_mean": 3.059607183786284e-05, "mean": 4.691841348858361e-05, "normalized": [ 1.7359303646705315, 1.6475859007944922, 1.5536416135247637, 1.4921074478374423, 1.436565604934822, 1.343834173349448, 1.2787439414688093, 1.2483315192759759, 1.191227587726315, 1.1603086071531024, 1.0995306735006558, 1.059977946939621, 1.0583984121521266, 1.0176934649371145, 0.9754909410255406, 0.9631621800279858, 0.9242301483750994, 0.8752327792179452, 0.8454160845678106, 0.8232909667825862, 0.7863453139125789, 0.77746685672799, 0.7497386589361914, 0.7388508716017261, 0.7093180450413769, 0.6908354487677191, 0.6685471901846501, 0.6592354484095886, 0.631440334976849, 0.6235901145414114, 0.6199506557967245, 0.6139807028410055 ], "peak_layer": 1, "peak_normalized": 1.7359303646705315, "population_cv": 0.3220412433027962, "rest_mean": 4.907019663530971e-05, "spike_contrast": 0.7193528234378936, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.529878449626267e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 8.144709863699973e-05, 7.730211655143648e-05, 7.289439963642508e-05, 7.000731420703232e-05, 6.740137905580923e-05, 6.305056740529835e-05, 5.999663699185476e-05, 5.856973439222202e-05, 5.5890508519951254e-05, 5.4439839004771784e-05, 5.1588234782684594e-05, 4.9732483603293076e-05, 4.965837433701381e-05, 4.774856279254891e-05, 4.576848732540384e-05, 4.519004141911864e-05, 4.3363412260077894e-05, 4.106453343410976e-05, 3.966558142565191e-05, 3.862750600092113e-05, 3.6894074582960457e-05, 3.647751145763323e-05, 3.517654840834439e-05, 3.4665710700210184e-05, 3.328007733216509e-05, 3.241290323785506e-05, 3.136717350571416e-05, 3.0930281354812905e-05, 2.9626178729813546e-05, 2.925785884144716e-05, 2.9087101211189292e-05, 2.8807000489905477e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 0.0001734125216898974, "first_to_last_ratio": 1.495517223041696, "imbalance_abs_log_ratio": 0.4024721155990803, "last_quartile_mean": 0.00011595488103921525, "mean": 0.00013990532806928968, "normalized": [ 1.3768889945483427, 1.3340940553978193, 1.2910049686488183, 1.2527203273458163, 1.2147536525239127, 1.1769854337196524, 1.14744939519443, 1.1220955988146075, 1.0944022472959734, 1.0773022658134643, 1.0508559212107247, 1.03206209473486, 1.0208171898483585, 1.000561882500425, 0.9805161605170362, 0.9663129296065233, 0.9487424991891081, 0.931931984738623, 0.9181974345490227, 0.9075104531838204, 0.8943640941785878, 0.8862664021756573, 0.8757830774574916, 0.86790401991425, 0.8562894022841879, 0.8478155487572495, 0.8387227683262297, 0.8314011142413483, 0.8224071457687988, 0.8161875050523302, 0.8112199678685613, 0.8064334645939688 ], "peak_layer": 1, "peak_normalized": 1.3768889945483427, "population_cv": 0.16325957486062354, "rest_mean": 0.0001431148236776978, "spike_contrast": 0.8564734849544942, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001225740517838858, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0001926341064972803, 0.00018664686649572104, 0.00018061847367789596, 0.00017526224837638438, 0.0001699505082797259, 0.00016466653323732316, 0.00016053428407758474, 0.00015698715287726372, 0.00015311270544771105, 0.00015072032692842185, 0.00014702034241054207, 0.00014439098595175892, 0.00014281776384450495, 0.00013998393842484802, 0.00013717943511437625, 0.00013519232743419707, 0.00013273413060232997, 0.00013038225006312132, 0.00012846071331296116, 0.00012696554767899215, 0.0001251263020094484, 0.00012399339175317436, 0.0001225267187692225, 0.00012142439663875848, 0.00011979944974882528, 0.00011861391249112785, 0.00011734178406186402, 0.00011631744564510882, 0.00011505914153531194, 0.00011418898066040128, 0.00011349399574100971, 0.00011282433843007311 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 0.00014675117381557357, "first_to_last_ratio": 1.2676837465168667, "imbalance_abs_log_ratio": 0.23719141364257146, "last_quartile_mean": 0.00011576323686313117, "mean": 0.00013042020850662084, "normalized": [ 1.169988069421413, 1.156175152925158, 1.1417665260225645, 1.1306726331060184, 1.1183621029407895, 1.1062694829504214, 1.0940115056230622, 1.0844989956600726, 1.0711591662817939, 1.0579765491085218, 1.0443900226505625, 1.035471661013805, 1.0260498632741444, 1.013992390086173, 1.0062479316494695, 0.9977001177294099, 0.9863163473759529, 0.9769959732660659, 0.9687585669786032, 0.9602628595878113, 0.9503845997255937, 0.9426104617668664, 0.9330243108845692, 0.9259760938667744, 0.914797625470175, 0.906289421438531, 0.8956357003805738, 0.888470885240542, 0.8806216558595509, 0.8756991509238615, 0.8712858841028995, 0.8681382926882502 ], "peak_layer": 1, "peak_normalized": 1.169988069421413, "population_cv": 0.09123006364857297, "rest_mean": 0.00013202972385678786, "spike_contrast": 0.9219804606102014, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012172882561571896, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00015259008796419948, 0.00015078860451467335, 0.00014890942838974297, 0.0001474625605624169, 0.00014585701865144074, 0.00014427989663090557, 0.00014268120867200196, 0.00014144058513920754, 0.00013970080181024969, 0.0001379815221298486, 0.00013620956451632082, 0.00013504642993211746, 0.0001338176371064037, 0.00013224509893916547, 0.00013123506505507976, 0.0001301202573813498, 0.00012863558367826045, 0.00012742001854348928, 0.00012634569429792464, 0.00012523768236860633, 0.00012394935765769333, 0.00012293545296415687, 0.00012168522516731173, 0.00012076599523425102, 0.00011930809705518186, 0.00011819845531135798, 0.00011680899478960782, 0.00011587455810513347, 0.00011485085997264832, 0.00011420886585256085, 0.00011363328667357564, 0.00011322277714498341 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00030002388848515693, "first_to_last_ratio": 2.2836575149168885, "imbalance_abs_log_ratio": 0.8257783308640597, "last_quartile_mean": 0.0001313786706305109, "mean": 0.0001947244400071213, "normalized": [ 1.8998833386658438, 1.7835645331795598, 1.6713073472446418, 1.5703972875779748, 1.4780250735507643, 1.3761669788219328, 1.3066447563403325, 1.2401013786965793, 1.1762106459027857, 1.150547637187856, 1.0768735950666006, 1.0264900078881174, 1.0140784159549234, 0.9666871251601796, 0.9096841069789114, 0.8754807117769676, 0.8421876125778792, 0.8073809902734768, 0.77911667677541, 0.7607651063686129, 0.7403822030704592, 0.7308855611321243, 0.7152791523187013, 0.7043381880055872, 0.6944157299089508, 0.686864629891504, 0.680599049276497, 0.6764772714226681, 0.6702412094778135, 0.6672258960044359, 0.6635863560708457, 0.6581114274310631 ], "peak_layer": 1, "peak_normalized": 1.8998833386658438, "population_cv": 0.3592853832280775, "rest_mean": 0.00020492727344390005, "spike_contrast": 0.6813594750078023, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013962913944851608, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0003699537192005664, 0.0003473036049399525, 0.0003254443872720003, 0.00030579473241232336, 0.00028780760476365685, 0.00026797334430739284, 0.0002544356684666127, 0.0002414780465187505, 0.0002290369593538344, 0.00022403974435292184, 0.00020969360775779933, 0.00019988269195891917, 0.0001974658516701311, 0.00018823760910890996, 0.00017713772831484675, 0.00017047749133780599, 0.00016399451124016196, 0.0001572168112033978, 0.00015171305858530104, 0.00014813955931458622, 0.00014417050988413393, 0.00014232128160074353, 0.00013928233238402754, 0.0001371518592350185, 0.00013521971413865685, 0.00013374933041632175, 0.00013252926873974502, 0.00013172665785532445, 0.00013051234418526292, 0.00012992518895771354, 0.00012921648158226162, 0.000128150379168801 ] } } }, "mlp_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "afe29f3ef4d2992353f0bb59fe7c5c2c5050fca671a53184825a8d63cd83a2bc", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.0027073738165199757, 0.002722575329244137, 0.0026850115973502398, 0.0026861224323511124, 0.00271838391199708, 0.0026645169127732515, 0.0026859452482312918, 0.0027359756641089916, 0.0026767742820084095, 0.0027451682835817337, 0.0026999011170119047, 0.002663193503394723, 0.002626648638397455, 0.0026531771291047335, 0.0026517712976783514, 0.002676903735846281, 0.002670086920261383, 0.0026355870068073273, 0.0026610493659973145, 0.0026388936676084995, 0.0026469926815479994, 0.00264862016774714, 0.002641494618728757, 0.002674198243767023, 0.002557571977376938, 0.00252733426168561, 0.0025468147359788418, 0.0026856842450797558, 0.002551089273765683, 0.0025778315030038357, 0.0026001501828432083, 0.0025662274565547705 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.00015858857113926206, "first_to_last_ratio": 3.757867690304466, "imbalance_abs_log_ratio": 1.3238516928952826, "last_quartile_mean": 4.220174423608114e-05, "mean": 8.573221236929385e-05, "normalized": [ 2.5461641896458365, 2.3032502265274117, 2.0924715218207024, 1.8886722264368014, 1.7031663195050915, 1.537434754996766, 1.4223955235407997, 1.3049501874408618, 1.2055297161045848, 1.1692542365736316, 1.0769571962730058, 1.0144505847411098, 0.9913601784485884, 0.9317001386730132, 0.857757082878185, 0.8155634736467603, 0.7682612945963089, 0.7217806418748633, 0.678338533281248, 0.6511820384364709, 0.6192882306865346, 0.6053793146733001, 0.5848721334223365, 0.5718144052517429, 0.5462281883927541, 0.527515083157495, 0.5112993585545178, 0.496920907994256, 0.47722358366421574, 0.46552107689700334, 0.4601234027970099, 0.45317424906679377 ], "peak_layer": 1, "peak_normalized": 2.5461641896458365, "population_cv": 0.5686091737198813, "rest_mean": 9.231269224848668e-05, "spike_contrast": 0.5437781067692293, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.0197621021652594e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00021828828903380781, 0.00019746273756027222, 0.00017939221288543195, 0.0001619200484128669, 0.00014601621660403907, 0.000131807682919316, 0.00012194511509733275, 0.00011187626660102978, 0.00010335272963857278, 0.00010024275252362713, 9.232992306351662e-05, 8.697109296917915e-05, 8.499150135321543e-05, 7.987671415321529e-05, 7.353741239057854e-05, 6.992006092332304e-05, 6.586474046343938e-05, 6.187985127326101e-05, 5.8155463193543255e-05, 5.582727681030519e-05, 5.309295011102222e-05, 5.1900507969548926e-05, 5.0142381951445714e-05, 4.902291402686387e-05, 4.682935104938224e-05, 4.522503513726406e-05, 4.3834825191879645e-05, 4.260212881490588e-05, 4.0913433622336015e-05, 3.991015182691626e-05, 3.944739728467539e-05, 3.885163096128963e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.00021242323236947414, "first_to_last_ratio": 1.8345651540958352, "imbalance_abs_log_ratio": 0.6068074801791611, "last_quartile_mean": 0.0001157894184871111, "mean": 0.00015111673860701558, "normalized": [ 1.771707915810251, 1.6366668487510905, 1.5242909365839221, 1.4186705636795862, 1.3264449439106374, 1.2453857912414694, 1.1893415085703798, 1.1330085309323608, 1.0858463851390703, 1.062861239127963, 1.023237806768965, 0.9942126959777651, 0.978436257513136, 0.9531027445378829, 0.9232414968561398, 0.904013237249231, 0.8853704564278231, 0.8680164030054869, 0.8515402788738432, 0.8395817783841374, 0.8267435189825344, 0.8185143642831039, 0.809172703595369, 0.8007918347797284, 0.7909896391756025, 0.7821052393452608, 0.7753388667989797, 0.7682453290925442, 0.7604995795745633, 0.7549901567959042, 0.7508572350862155, 0.7467737131490535 ], "peak_layer": 1, "peak_normalized": 1.771707915810251, "population_cv": 0.2714170004285523, "rest_mean": 0.00015645500981783562, "spike_contrast": 0.7816309251520459, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012229007406858728, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0002677347220014781, 0.00024732775636948645, 0.00023034587502479553, 0.00021438486874103546, 0.00020044803386554122, 0.0001881986390799284, 0.00017972940986510366, 0.00017121655400842428, 0.00016408956435043365, 0.00016061612404882908, 0.0001546283601783216, 0.00015024218009784818, 0.00014785809617023915, 0.0001440297783119604, 0.00013951724395155907, 0.000136611532070674, 0.0001337942958343774, 0.00013117180787958205, 0.0001286819897359237, 0.00012687486014328897, 0.00012493478425312787, 0.00012369122123345733, 0.00012227953993715346, 0.00012101305037504062, 0.00011953177454415709, 0.00011818919301731512, 0.00011716668086592108, 0.00011609472858253866, 0.00011492421617731452, 0.00011409165017539635, 0.00011346709652571008, 0.00011285000800853595 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.18837904930114746, "first_to_last_ratio": 1.8345651581276443, "imbalance_abs_log_ratio": 0.6068074823768531, "last_quartile_mean": 0.10268321540206671, "mean": 0.13401183625683188, "normalized": [ 1.7717078885231814, 1.6366668119471057, 1.524290957183596, 1.418670514273678, 1.326444920903592, 1.2453858718827995, 1.1893415500975306, 1.1330085716119305, 1.0858463365841577, 1.062861212873729, 1.0232377434953237, 0.9942127340740732, 0.9784362443707608, 0.9531027271661117, 0.9232414917376033, 0.904013241653558, 0.885370477646398, 0.8680164390463408, 0.8515402681847718, 0.8395816969955481, 0.8267435345782925, 0.8185144279720658, 0.8091727254708025, 0.8007918405918669, 0.7909896328038464, 0.7821052665924377, 0.7753388466443417, 0.7682453528580236, 0.7604996016741158, 0.7549901615676585, 0.7508572336421532, 0.7467736753526057 ], "peak_layer": 1, "peak_normalized": 1.7717078885231814, "population_cv": 0.27141699472708714, "rest_mean": 0.1387458680956452, "spike_contrast": 0.7816309473985974, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.10844806432724, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.23742982745170593, 0.2193327248096466, 0.2042730301618576, 0.19011864066123962, 0.1777593195438385, 0.16689644753932953, 0.15938584506511688, 0.151836559176445, 0.1455162614583969, 0.1424359828233719, 0.13712596893310547, 0.13323627412319183, 0.13112203776836395, 0.12772704660892487, 0.1237252876162529, 0.1211484745144844, 0.11865012347698212, 0.11632447689771652, 0.11411647498607635, 0.11251388490200043, 0.11079341918230057, 0.10969062149524689, 0.10843872278928757, 0.10731558501720428, 0.10600197315216064, 0.10481136292219162, 0.10390458256006241, 0.1029539704322815, 0.10191594809293747, 0.10117761790752411, 0.10062375664710999, 0.10007651150226593 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 6.873187476230669e-05, "first_to_last_ratio": 2.248757500145603, "imbalance_abs_log_ratio": 0.8103778415280068, "last_quartile_mean": 3.0564378221242805e-05, "mean": 4.685761763312257e-05, "normalized": [ 1.7400006587426948, 1.6405194725375354, 1.5551306973978087, 1.4944078341288562, 1.4327724019957897, 1.3414187773842097, 1.2804088050749736, 1.2499327782691996, 1.1927955171655855, 1.1608931065609767, 1.0981756972550076, 1.0577423904261343, 1.0542786038595535, 1.0218640087393278, 0.9775807389332813, 0.9570972402588515, 0.9209982861589985, 0.8815092249082567, 0.8478232134430647, 0.8204952153707635, 0.7871780668180614, 0.7759097745574741, 0.7506963524238334, 0.7421155343193898, 0.7072927325438048, 0.6891814936654451, 0.6695399899996352, 0.6616343983211914, 0.6345111343049281, 0.6212687927046295, 0.6200231913288072, 0.6148038704019305 ], "peak_layer": 1, "peak_normalized": 1.7400006587426948, "population_cv": 0.3217536161653951, "rest_mean": 4.900405669845727e-05, "spike_contrast": 0.719671983430414, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.526684668031521e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 8.153228554874659e-05, 7.687083416385576e-05, 7.286971958819777e-05, 7.002439087955281e-05, 6.71363013680093e-05, 6.285568815656006e-05, 5.999690620228648e-05, 5.8568872191244736e-05, 5.5891556257847697e-05, 5.439668530016206e-05, 5.145789691596292e-05, 4.9563288484932855e-05, 4.9400983698433265e-05, 4.7882112994557247e-05, 4.580710447044112e-05, 4.484729652176611e-05, 4.3155785533599555e-05, 4.130542220082134e-05, 3.972697595600039e-05, 3.844645107164979e-05, 3.688528886414133e-05, 3.635728353401646e-05, 3.517584264045581e-05, 3.477376594673842e-05, 3.314205241622403e-05, 3.229340290999971e-05, 3.1373048841487616e-05, 3.10026116494555e-05, 2.9731680115219206e-05, 2.9111175535945222e-05, 2.9052809622953646e-05, 2.8808244678657502e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 0.0001719075116852764, "first_to_last_ratio": 1.4894635014227988, "imbalance_abs_log_ratio": 0.39841598895878805, "last_quartile_mean": 0.00011541572621354135, "mean": 0.00013894635094402474, "normalized": [ 1.3738985779819257, 1.3296082759737424, 1.2883384004275333, 1.2499940655466777, 1.2130730176151365, 1.1755503979222366, 1.147409009170091, 1.11990595257071, 1.0929037170303153, 1.075598370036652, 1.0512762916941902, 1.0312594730951, 1.0194418974669357, 1.0018996501730997, 0.9803460168062212, 0.9648004557293595, 0.9493010807852901, 0.9344788929945884, 0.9198953859218769, 0.908001147597105, 0.8952913731815082, 0.886953676542665, 0.8771894463102521, 0.8683888934041172, 0.8579399877904575, 0.8482799643655919, 0.8408636339472488, 0.8330870307470335, 0.82434015220157, 0.8181187963300758, 0.8135579418994486, 0.809009026741244 ], "peak_layer": 1, "peak_normalized": 1.3738985779819257, "population_cv": 0.16161423472023242, "rest_mean": 0.0001421073115866252, "spike_contrast": 0.8576417505420824, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012187716347398237, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0001908981939777732, 0.0001847442181315273, 0.00017900991952046752, 0.00017368211410939693, 0.00016855206922627985, 0.00016333843814209104, 0.00015942829486448318, 0.00015560684551019222, 0.0001518549834145233, 0.00014945046859793365, 0.00014607100456487387, 0.0001432897406630218, 0.00014164773165248334, 0.00013921030040364712, 0.000136215501697734, 0.0001340555027127266, 0.0001319019211223349, 0.00012984243221580982, 0.0001278161071240902, 0.00012616344611160457, 0.00012439746933523566, 0.00012323897681199014, 0.00012188227265141904, 0.00012065946793882176, 0.00011920763063244522, 0.00011786540562752634, 0.00011683493357850239, 0.00011575440294109285, 0.0001145390560850501, 0.00011367462138878182, 0.00011304090730845928, 0.00011240885214647278 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 0.00014552221000485588, "first_to_last_ratio": 1.262697055715921, "imbalance_abs_log_ratio": 0.2332499537229619, "last_quartile_mean": 0.00011524712863320019, "mean": 0.00012953460168318998, "normalized": [ 1.1676213631634658, 1.1525055688027706, 1.1391580058081985, 1.1289307982409096, 1.1166731577821778, 1.1054001772932092, 1.094813144753513, 1.0822854389705017, 1.0679308603491235, 1.0564894822224282, 1.0456838360716472, 1.0337072702557388, 1.0234945545465333, 1.0158819477934218, 1.0051772952892828, 0.9951467950359763, 0.9866643397779161, 0.9787559418439278, 0.970448736962688, 0.9607908684400951, 0.9509939230158972, 0.9428061355388626, 0.934924980051271, 0.9261034265480621, 0.9170582603044295, 0.9058562786860701, 0.8981778407016658, 0.8907564927732574, 0.8831168120837269, 0.8778216107141017, 0.8745192645108896, 0.8703053916682415 ], "peak_layer": 1, "peak_normalized": 1.1676213631634658, "population_cv": 0.08967236486753827, "rest_mean": 0.00013110875065817877, "spike_contrast": 0.9231588022206524, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012103419721825049, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00015124736819416285, 0.00014928934979252517, 0.000147560378536582, 0.00014623560127802193, 0.00014464781270362437, 0.00014318757166620344, 0.00014181618462316692, 0.00014019341324456036, 0.0001383339986205101, 0.00013685194426216185, 0.00013545223919209093, 0.00013390085950959474, 0.00013257795944809914, 0.00013159186346456409, 0.00013020524056628346, 0.00012890594371128827, 0.00012780717224813998, 0.00012678276107180864, 0.0001257066905964166, 0.0001244556624442339, 0.00012318661902099848, 0.0001221260172314942, 0.00012110513489460573, 0.0001199624384753406, 0.00011879077646881342, 0.00011733973224181682, 0.00011634510883595794, 0.00011538378748809919, 0.0001143941844929941, 0.00011370827269274741, 0.00011328050459269434, 0.0001127346622524783 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00029741139769612346, "first_to_last_ratio": 2.278072415841521, "imbalance_abs_log_ratio": 0.8233296537390045, "last_quartile_mean": 0.00013055396993877366, "mean": 0.00019328556300024502, "normalized": [ 1.889398576482247, 1.780233894901915, 1.6772594716902607, 1.566224958959574, 1.4762232610465842, 1.375571144634086, 1.3083686481730985, 1.2364401527267181, 1.173148897934763, 1.1508864184295666, 1.080457040307468, 1.0318582947152708, 1.0081957004611959, 0.9698994715426978, 0.9066127339785177, 0.8754550048258108, 0.8446146862114313, 0.8115724449181009, 0.7806677558079268, 0.7613480996239201, 0.7403134026279838, 0.7294052762839086, 0.7160186976339193, 0.7062575707090604, 0.6942777320753531, 0.6866005528973866, 0.6829938476675228, 0.6753403086482458, 0.6715967314167978, 0.6686860556999406, 0.6640286432895989, 0.6600445236791291 ], "peak_layer": 1, "peak_normalized": 1.889398576482247, "population_cv": 0.3579154260008552, "rest_mean": 0.00020340604748559632, "spike_contrast": 0.681567477924495, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013863494677934796, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0003651934675872326, 0.0003440935106482357, 0.00032419004128314555, 0.0003027286729775369, 0.0002853326441254467, 0.00026587804313749075, 0.00025288877077400684, 0.00023898603103589267, 0.00022675274522043765, 0.00022244972933549434, 0.0002088367473334074, 0.00019944331143051386, 0.00019486967357806861, 0.00018746756541077048, 0.00017523515271022916, 0.00016921281348913908, 0.0001632518251426518, 0.00015686523693148047, 0.00015089180669747293, 0.00014715759607497603, 0.00014309189282357693, 0.00014098350948188454, 0.00013839607709087431, 0.0001365093921776861, 0.0001341938623227179, 0.00013270997442305088, 0.00013201285037212074, 0.00013053353177383542, 0.0001298099523410201, 0.00012924736074637622, 0.00012834715016651899, 0.00012757707736454904 ] } } }, "post_attention_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "31853ffb07b24cc59e774bb20656840b4358b3080b42516d380f2dc0f46404c8", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.0022265110164880753, 0.0040087271481752396, 0.005379301495850086, 0.006410556845366955, 0.0027253651060163975, 0.004838948138058186, 0.006175771821290255, 0.00742313964292407, 0.0030937574338167906, 0.005150026176124811, 0.007019025273621082, 0.008143599145114422, 0.004255065228790045, 0.0066224560141563416, 0.008129785768687725, 0.010099578648805618, 0.0041999719105660915, 0.00719039561226964, 0.009030314162373543, 0.010736005380749702, 0.005131533369421959, 0.00787449348717928, 0.009912566281855106, 0.012113703414797783, 0.0059690922498703, 0.00876442901790142, 0.010806876234710217, 0.012353328987956047, 0.006502739619463682, 0.009470963850617409, 0.012123330496251583, 0.01384514570236206 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.00015962236011546338, "first_to_last_ratio": 3.7698422272252903, "imbalance_abs_log_ratio": 1.3270331510418396, "last_quartile_mean": 4.2341920561739244e-05, "mean": 8.61727822893954e-05, "normalized": [ 2.542734798705971, 2.3221448837042376, 2.101145461274622, 1.8911858524935854, 1.7004204056817789, 1.5354142426015858, 1.4229001770142031, 1.3028733810328659, 1.2049412416934622, 1.1705258152816296, 1.0773908232649334, 1.0133220774376726, 0.990609360100461, 0.9272534949358001, 0.8550385909364457, 0.8185651575045715, 0.7703972051466339, 0.7173947009730917, 0.6760256695667757, 0.6514523109028573, 0.618419077603279, 0.6053620388084041, 0.5838930841859182, 0.5697043695032278, 0.5457802700901755, 0.5276149005446632, 0.5104701332743111, 0.49517988582787786, 0.47527562953524777, 0.46579059316028215, 0.45896435764104826, 0.45181000957238127 ], "peak_layer": 1, "peak_normalized": 2.542734798705971, "population_cv": 0.5705202801130907, "rest_mean": 9.280119640472414e-05, "spike_contrast": 0.542873885449774, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.03793460666202e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00021911453222855926, 0.00020010568550787866, 0.00018106155039276928, 0.00016296874673571438, 0.00014652995741926134, 0.00013231091725174338, 0.0001226152671733871, 0.00011227222421439365, 0.00010383313929196447, 0.00010086746624438092, 9.284176485380158e-05, 8.732078276807442e-05, 8.536356472177431e-05, 7.99040135461837e-05, 7.368105434579775e-05, 7.053803710732609e-05, 6.638727063545957e-05, 6.181989738252014e-05, 5.825501284562051e-05, 5.613745815935545e-05, 5.329089253791608e-05, 5.216573117650114e-05, 5.0315691623836756e-05, 4.9093010602518916e-05, 4.703140439232811e-05, 4.546604395727627e-05, 4.398863165988587e-05, 4.267102849553339e-05, 4.095582335139625e-05, 4.013847137684934e-05, 3.955023566959426e-05, 3.893372559105046e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.00021442477373057045, "first_to_last_ratio": 1.8431254562763404, "imbalance_abs_log_ratio": 0.6114627481432037, "last_quartile_mean": 0.00011633759004325839, "mean": 0.00015221248349917005, "normalized": [ 1.7740805953138137, 1.6484457246979003, 1.5301434809496548, 1.4217241600875643, 1.3264958046529274, 1.2454577490392524, 1.1897861330408546, 1.1336265611690741, 1.086914920172498, 1.064301358569379, 1.0227209980571423, 0.9941753882589759, 0.9788979908211966, 0.9511107967643203, 0.9228143274453928, 0.9052447611165039, 0.8846530997418666, 0.8652930871753776, 0.8497381552743479, 0.8388656494639406, 0.8256334792987466, 0.8176309647342147, 0.8076951735882674, 0.8000661307359944, 0.7892605467734257, 0.78144613040264, 0.7731219607566042, 0.7664988488799842, 0.7584935618388989, 0.7529639051546407, 0.7485195296370439, 0.7441790263875565 ], "peak_layer": 1, "peak_normalized": 1.7740805953138137, "population_cv": 0.2736411578809574, "rest_mean": 0.00015762286892384981, "spike_contrast": 0.7803207938393821, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012299640220589936, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00027003721334040165, 0.00025091401766985655, 0.00023290693934541196, 0.0002164041652576998, 0.000201909220777452, 0.0001895742170745507, 0.00018110030214302242, 0.00017255211423616856, 0.00016544201935175806, 0.00016199995297938585, 0.00015567090304102749, 0.00015132590488065034, 0.0001490004942752421, 0.00014477093645837158, 0.00014046386058907956, 0.00013778955326415598, 0.0001346552453469485, 0.00013170840975362808, 0.00012934075493831187, 0.00012768582382705063, 0.00012567172234412283, 0.00012445363972801715, 0.00012294128828216344, 0.00012178005272289738, 0.00012013530795229599, 0.00011894585622940212, 0.00011767881369451061, 0.00011667069338727742, 0.00011545218876563013, 0.00011461050598882139, 0.00011393401655368507, 0.00011327333777444437 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.190154030919075, "first_to_last_ratio": 1.84312545379361, "imbalance_abs_log_ratio": 0.6114627467961817, "last_quartile_mean": 0.10316933691501617, "mean": 0.13498355145566165, "normalized": [ 1.774080628234443, 1.6484456620184311, 1.5301434020504485, 1.4217242444591272, 1.326495766996536, 1.2454576834375484, 1.189786122599316, 1.133626627255238, 1.0869149569673349, 1.0643014004698326, 1.0227209907477024, 0.9941753870542451, 0.9788979589698585, 0.9511107512164827, 0.9228143032197887, 0.9052447972745712, 0.8846531331847195, 0.8652931173486506, 0.8497382179112226, 0.8388656682549475, 0.8256334805754935, 0.8176309131099365, 0.8076951535479161, 0.8000661540388709, 0.7892605573866218, 0.7814461538045975, 0.7731219580171443, 0.7664988542586116, 0.7584935821787124, 0.7529638601652359, 0.7485195164462104, 0.7441789968002049 ], "peak_layer": 1, "peak_normalized": 1.774080628234443, "population_cv": 0.27364115345546325, "rest_mean": 0.13978153549962574, "spike_contrast": 0.7803207857775153, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.10907443761825561, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.2394717037677765, 0.22251304984092712, 0.2065441906452179, 0.19190938770771027, 0.1790551096200943, 0.16811630129814148, 0.16060155630111694, 0.1530209481716156, 0.14671564102172852, 0.1436631828546524, 0.13805051147937775, 0.13419732451438904, 0.13213512301445007, 0.1283843070268631, 0.1245647519826889, 0.12219315767288208, 0.11941362172365189, 0.11680033802986145, 0.11470068246126175, 0.1132330670952797, 0.11144693940877914, 0.11036672443151474, 0.10902556031942368, 0.10799577087163925, 0.10653719305992126, 0.10548237711191177, 0.1043587476015091, 0.10346473753452301, 0.10238415747880936, 0.10163773596286774, 0.10103782266378403, 0.10045192390680313 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 6.883367768750759e-05, "first_to_last_ratio": 2.249759357551583, "imbalance_abs_log_ratio": 0.810823258297238, "last_quartile_mean": 3.059601795030176e-05, "mean": 4.691846504556452e-05, "normalized": [ 1.7359059710084281, 1.6475789727938472, 1.5536456441204052, 1.4921194549590373, 1.4365657321885223, 1.3438513058553838, 1.2787343947807137, 1.2483299149152907, 1.1912354282513808, 1.1603083401306369, 1.099547144002948, 1.059972440022979, 1.0583955432739494, 1.0176800955770247, 0.9754963047746087, 0.9631595708780456, 0.9242400656751482, 0.8752301116137658, 0.8454245377074081, 0.8232853322610043, 0.7863494898171388, 0.7774642190168813, 0.7497486129019063, 0.7388483538626245, 0.709315327139865, 0.6908343794807076, 0.6685469983108738, 0.6592365073812129, 0.6314341746588822, 0.6235898169926835, 0.6199474545603838, 0.6139783610863121 ], "peak_layer": 1, "peak_normalized": 1.7359059710084281, "population_cv": 0.3220406003070692, "rest_mean": 4.9070239684725595e-05, "spike_contrast": 0.7193541792517962, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.52988819940947e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 8.144604362314567e-05, 7.730187644483522e-05, 7.28946688468568e-05, 7.000795449130237e-05, 6.740145909134299e-05, 6.305144052021205e-05, 5.999625500408001e-05, 5.8569723478285596e-05, 5.589093780145049e-05, 5.4439886298496276e-05, 5.158906424185261e-05, 4.973227987647988e-05, 4.9658294301480055e-05, 4.774798799189739e-05, 4.576878927764483e-05, 4.51899686595425e-05, 4.3363925215089694e-05, 4.1064453398576006e-05, 3.9666021621087566e-05, 3.86272840842139e-05, 3.689431105158292e-05, 3.6477427784120664e-05, 3.5177054087398574e-05, 3.466563066467643e-05, 3.3279986382694915e-05, 3.2412888685939834e-05, 3.136719897156581e-05, 3.093036502832547e-05, 2.9625922252307646e-05, 2.9257877031341195e-05, 2.9086982976878062e-05, 2.8806922273361124e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 0.00017341253987979144, "first_to_last_ratio": 1.4955148931299893, "imbalance_abs_log_ratio": 0.4024705576674927, "last_quartile_mean": 0.00011595507385209203, "mean": 0.00013990553861731314, "normalized": [ 1.3768818258178377, 1.3340879911950096, 1.2910054180628048, 1.2527236427111486, 1.2147542166907819, 1.1769889670731195, 1.1474411155803916, 1.1220953663134672, 1.0944075691302455, 1.0773007485615413, 1.0508678613623372, 1.0320596054419082, 1.0208161736522385, 1.000559648637946, 0.9805197815181176, 0.9663128275349431, 0.948744191771126, 0.9319305822463463, 0.9182019814348666, 0.9075038868226692, 0.8943650364968407, 0.886262676120959, 0.8757907045344768, 0.8679002174812178, 0.856288373659771, 0.8478178092769747, 0.8387215581135022, 0.8313975747674917, 0.8224084044020752, 0.8161864847722404, 0.8112200992010112, 0.8064376596145924 ], "peak_layer": 1, "peak_normalized": 1.3768818258178377, "population_cv": 0.16325888721442564, "rest_mean": 0.0001431150389921472, "spike_contrast": 0.8564734877369171, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001225742365932092, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0001926333934534341, 0.00018664629897102714, 0.00018061880837194622, 0.0001752629759721458, 0.00016995084297377616, 0.0001646672753849998, 0.00016053336730692536, 0.00015698735660407692, 0.00015311368042603135, 0.00015072034148033708, 0.00014702223415952176, 0.00014439085498452187, 0.0001428178366040811, 0.00013998383656144142, 0.00013718014815822244, 0.00013519251660909504, 0.00013273456715978682, 0.00013038225006312132, 0.00012846154277212918, 0.00012696482008323073, 0.00012512662215158343, 0.0001239930570591241, 0.00012252797023393214, 0.000121424047392793, 0.00011979948612861335, 0.00011861440725624561, 0.00011734179133782163, 0.0001163171255029738, 0.00011505949078127742, 0.00011418900976423174, 0.00011349418491590768, 0.00011282509512966499 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 0.00014675183047074825, "first_to_last_ratio": 1.2676545313577883, "imbalance_abs_log_ratio": 0.23716836728277432, "last_quartile_mean": 0.00011576642282307148, "mean": 0.00013041823922321782, "normalized": [ 1.1700698937843759, 1.1560440994564782, 1.1417398043557874, 1.130463200949706, 1.118403537277814, 1.1065304334184665, 1.0939881912841023, 1.0846815122144253, 1.0709468270677047, 1.0579907390333845, 1.0442838370166418, 1.035550673160169, 1.025959021745254, 1.0140203095307772, 1.0062047700154308, 0.9976631870190885, 0.9861765922450596, 0.9770016877798172, 0.9687702939454803, 0.9602422119882037, 0.9504995944091904, 0.9425883202531964, 0.9330014667281973, 0.9259385264566372, 0.9147443798098084, 0.906202796813466, 0.8958065504175365, 0.8885655303249116, 0.8806194435723184, 0.8757905905296257, 0.8713783728577705, 0.8681336045391749 ], "peak_layer": 1, "peak_normalized": 1.1700698937843759, "population_cv": 0.09121429133712369, "rest_mean": 0.0001320278307604086, "spike_contrast": 0.921975648780332, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001217264449223876, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0001525984553154558, 0.0001507692359155044, 0.000148903694935143, 0.00014743302017450333, 0.00014586022007279098, 0.00014431175077334046, 0.00014267601363826543, 0.00014146225294098258, 0.000139670999487862, 0.00013798128929920495, 0.00013619365927297622, 0.00013505469541996717, 0.00013380376913119107, 0.00013224674330558628, 0.00013122745440341532, 0.00013011347618885338, 0.00012861541472375393, 0.00012741883983835578, 0.0001263453159481287, 0.0001252330985153094, 0.0001239624834852293, 0.00012293070903979242, 0.00012168040848337114, 0.00012075927224941552, 0.00011929935135412961, 0.00011818537313956767, 0.00011682951299007982, 0.00011588515189941972, 0.00011484883725643158, 0.00011421906674513593, 0.00011364363308530301, 0.00011322045611450449 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00030004180916876066, "first_to_last_ratio": 2.2837506327398005, "imbalance_abs_log_ratio": 0.8258191057718037, "last_quartile_mean": 0.00013138116082700435, "mean": 0.00019473525253488333, "normalized": [ 1.9000096511818072, 1.783700741635724, 1.6708066910504589, 1.570366884717585, 1.4783058799917985, 1.376185321596543, 1.3064973297630273, 1.2402700043336092, 1.1767953849206394, 1.1505341943958838, 1.0771542571420398, 1.026098162637436, 1.013898063759585, 0.9669473025647412, 0.9095398902231254, 0.8755205777387188, 0.842301214161826, 0.8073124727024382, 0.7791991818813049, 0.760574607765185, 0.7402055397447304, 0.7308143413325056, 0.7152667122155245, 0.7043714155678553, 0.6942467749765324, 0.6868749151283189, 0.680655564574882, 0.676363713518914, 0.6701061029149729, 0.6672276319278244, 0.6636177363814285, 0.6582317375530352 ], "peak_layer": 1, "peak_normalized": 1.9000096511818072, "population_cv": 0.35930746026536914, "rest_mean": 0.00020494150898971216, "spike_contrast": 0.6812747128050888, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013962146767880766, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0003699988592416048, 0.0003473494143690914, 0.0003253649629186839, 0.0003058057918678969, 0.0002878782688640058, 0.0002679917961359024, 0.0002544210874475539, 0.00024152429250534624, 0.00022916354646440595, 0.000224049566895701, 0.00020975990628357977, 0.00019981748482678086, 0.000197441695490852, 0.0001882987271528691, 0.00017711948021315038, 0.00017049472080543637, 0.00016402573965024203, 0.00015721219824627042, 0.00015173754945863038, 0.00014811068831477314, 0.00014414411270990968, 0.0001423153153154999, 0.0001392876438330859, 0.00013716594548895955, 0.00013519432104658335, 0.00013375876005738974, 0.00013254763325676322, 0.0001317118585575372, 0.00013049328117631376, 0.00012993274140171707, 0.00012922976748086512, 0.00012818092363886535 ] } } }, "post_mlp_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "7fd9c911e93175896ad618842c120e77b3f017c17b7eec9c2bfb01fc6dbc8ac2", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.0034885562490671873, 0.004831995815038681, 0.005952076520770788, 0.006925336085259914, 0.00385080324485898, 0.005542940925806761, 0.006729288958013058, 0.007927131839096546, 0.004081308841705322, 0.005811207927763462, 0.007534808944910765, 0.008581922389566898, 0.00499920267611742, 0.007102519739419222, 0.008542952127754688, 0.010447698645293713, 0.005041283555328846, 0.007636935915797949, 0.009387043304741383, 0.011020516976714134, 0.005807018838822842, 0.008367395959794521, 0.01024632342159748, 0.012378672137856483, 0.006453148555010557, 0.009127611294388771, 0.011037957854568958, 0.012611051090061665, 0.007010402157902718, 0.009794782847166061, 0.012445803731679916, 0.014087481424212456 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.00015858902861509705, "first_to_last_ratio": 3.757869257587208, "imbalance_abs_log_ratio": 1.3238521099622325, "last_quartile_mean": 4.220184837322449e-05, "mean": 8.57325238712292e-05, "normalized": [ 2.5462257183789148, 2.3032637538343064, 2.0924289533379263, 1.8886492391629932, 1.7031256747388024, 1.5374176267930122, 1.4224231993460965, 1.3049597038569594, 1.2055618292025094, 1.169245405308266, 1.0769812048488787, 1.0144334047898542, 0.9913512297333369, 0.9317090592952362, 0.8577499774884012, 0.81555804918997, 0.7682725064221458, 0.7217807351250844, 0.6783324192661732, 0.6511800118936032, 0.6192916242842709, 0.6053728292385717, 0.5848720876072855, 0.5718124973482859, 0.5462302773879167, 0.5275173250132701, 0.5112998770983732, 0.49691833866078555, 0.47721658788927634, 0.46551989467389665, 0.46012635628943105, 0.45317260249616614 ], "peak_layer": 1, "peak_normalized": 2.5462257183789148, "population_cv": 0.5686111305901927, "rest_mean": 9.231303704803361e-05, "spike_contrast": 0.5437775023084314, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.019775271648541e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00021829435718245804, 0.00019746461475733668, 0.0001793892151908949, 0.0001619186659809202, 0.00014601326256524771, 0.00013180669338908046, 0.00012194793089292943, 0.00011187748896190897, 0.00010335585830034688, 0.00010024235962191597, 9.233231685357168e-05, 8.69699360919185e-05, 8.499104296788573e-05, 7.987776916706935e-05, 7.353707042057067e-05, 6.991984992055222e-05, 6.586594099644572e-05, 6.188008410390466e-05, 5.8155150327365845e-05, 5.582730591413565e-05, 5.309343396220356e-05, 5.190014053368941e-05, 5.014256021240726e-05, 4.90229285787791e-05, 4.682970029534772e-05, 4.522539165918715e-05, 4.383502891869284e-05, 4.2602063331287354e-05, 4.0912982512963936e-05, 3.991019548266195e-05, 3.944779382436536e-05, 3.885163096128963e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.00021242368711682502, "first_to_last_ratio": 1.8345658968465297, "imbalance_abs_log_ratio": 0.6068078850437921, "last_quartile_mean": 0.0001157896194854402, "mean": 0.0001511171253696375, "normalized": [ 1.7717451736709768, 1.636674793183293, 1.5242755762025908, 1.4186515254985965, 1.3264266232459345, 1.2453738409548603, 1.1893512719369177, 1.1330135273991528, 1.085860457802012, 1.0628542818802846, 1.023252039669589, 0.9942070699734702, 0.9784308644775869, 0.9531083940389613, 0.9232390376562555, 0.9040072643238835, 0.8853775311228679, 0.8680181295617843, 0.8515367513400655, 0.8395795332993111, 0.8267439067367663, 0.8185133286645813, 0.8091730400239778, 0.800791325997512, 0.7909899739933552, 0.7821072339266707, 0.7753403490721149, 0.7682422073314642, 0.7604971517068638, 0.7549863467424895, 0.7508596466770359, 0.7467718018887739 ], "peak_layer": 1, "peak_normalized": 1.7717451736709768, "population_cv": 0.2714186609524326, "rest_mean": 0.00015645541295978345, "spike_contrast": 0.7816308178118705, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012229037238284946, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0002677410375326872, 0.0002473295899108052, 0.00023034414334688336, 0.00021438254043459892, 0.00020044577831868082, 0.00018819731485564262, 0.00017973134526982903, 0.000171217747265473, 0.0001640921109355986, 0.00016061548376455903, 0.00015463090676348656, 0.00015024171443656087, 0.0001478576596127823, 0.00014403100067283958, 0.00013951722939964384, 0.00013661097909789532, 0.00013379570737015456, 0.0001311724045081064, 0.0001286817860091105, 0.00012687484559137374, 0.0001249351626029238, 0.00012369138130452484, 0.00012227990373503417, 0.00012101328320568427, 0.00011953213106608018, 0.0001181897969217971, 0.0001171672047348693, 0.00011609455395955592, 0.00011492414341773838, 0.0001140913664130494, 0.00011346775136189535, 0.00011285000800853595 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.18837944976985455, "first_to_last_ratio": 1.8345658800697515, "imbalance_abs_log_ratio": 0.6068078758989703, "last_quartile_mean": 0.10268339328467846, "mean": 0.1340121782850474, "normalized": [ 1.7717451751532867, 1.6366747548098712, 1.5242756140248972, 1.4186514377480737, 1.3264266357207033, 1.2453739091726768, 1.1893513017871604, 1.1330134634090196, 1.0858604665419038, 1.0628542749024097, 1.0232520332438115, 0.9942070832344971, 0.9784308561813967, 0.9531084116998004, 0.9232390798282929, 0.9040072094634526, 0.8853775581660035, 0.8680181154203164, 0.8515367049614931, 0.8395794986042229, 0.8267439263823189, 0.8185133396785964, 0.809173050926764, 0.8007913534932437, 0.7909900046670136, 0.782107217826699, 0.7753403703776178, 0.7682422246066585, 0.7604971603471317, 0.7549863443945886, 0.7508596538022533, 0.7467717694238248 ], "peak_layer": 1, "peak_normalized": 1.7717451751532867, "population_cv": 0.27141865758333467, "rest_mean": 0.13874622461972413, "spike_contrast": 0.7816308398662988, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.10844832807779312, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.23743543028831482, 0.21933434903621674, 0.20427149534225464, 0.19011656939983368, 0.17775732278823853, 0.16689527034759521, 0.15938755869865417, 0.15183760225772858, 0.14551852643489838, 0.14243541657924652, 0.13712823390960693, 0.1332358568906784, 0.1311216503381729, 0.12772813439369202, 0.1237252801656723, 0.12114797532558441, 0.1186513751745224, 0.1163249984383583, 0.11411628872156143, 0.11251387745141983, 0.11079375445842743, 0.10969075560569763, 0.10843904316425323, 0.107315793633461, 0.10600229352712631, 0.104811891913414, 0.10390505194664001, 0.10295381397008896, 0.1019158810377121, 0.10117736458778381, 0.10062433779239655, 0.10007651150226593 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 6.873207303215167e-05, "first_to_last_ratio": 2.248757814140914, "imbalance_abs_log_ratio": 0.8103779811585758, "last_quartile_mean": 3.0564462122129044e-05, "mean": 4.6857780716891284e-05, "normalized": [ 1.7400119939228862, 1.6405409364325143, 1.555121247708426, 1.4944054279879166, 1.4327475398601994, 1.3414100714975594, 1.2804101716466747, 1.2499370459040815, 1.192804641985012, 1.1608858830017554, 1.098191983597808, 1.057744066133373, 1.0542717513613309, 1.021869458338179, 0.9775749297615711, 0.9570927445976433, 0.9210091333285576, 0.8815125232807631, 0.847820728512489, 0.8204970180441978, 0.7871838673851995, 0.7759041238100245, 0.7506938949785397, 0.7421070509174572, 0.7072987335050381, 0.6891811912847667, 0.6695407652827761, 0.6616333377893114, 0.6345027924930273, 0.6212629426015038, 0.6200302724102036, 0.614801730639213 ], "peak_layer": 1, "peak_normalized": 1.7400119939228862, "population_cv": 0.32175451551289735, "rest_mean": 4.9004235565748616e-05, "spike_contrast": 0.7196709452950106, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.5266924533061685e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 8.153310045599937e-05, 7.687210745643824e-05, 7.28695304132998e-05, 7.002452184678987e-05, 6.713537004543468e-05, 6.28554989816621e-05, 5.9997179050697014e-05, 5.8569276006892323e-05, 5.589217835222371e-05, 5.439653614303097e-05, 5.1458839152473956e-05, 4.9563539505470544e-05, 4.9400834541302174e-05, 4.788253500009887e-05, 4.58069916930981e-05, 4.4847241952084005e-05, 4.315644400776364e-05, 4.130572051508352e-05, 3.972699778387323e-05, 3.8446669350378215e-05, 3.688568904181011e-05, 3.635714529082179e-05, 3.5175849916413426e-05, 3.477348946034908e-05, 3.3142448955914006e-05, 3.2293501135427505e-05, 3.13731943606399e-05, 3.1002669857116416e-05, 2.9731392714893445e-05, 2.9111002731951885e-05, 2.905324254243169e-05, 2.8808244678657502e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 0.00017190765356644988, "first_to_last_ratio": 1.4894621368061003, "imbalance_abs_log_ratio": 0.3984150727783497, "last_quartile_mean": 0.00011541592721187044, "mean": 0.00013894661265112518, "normalized": [ 1.3739040544627474, 1.3296102750467884, 1.2883342981472405, 1.2499918159027226, 1.2130627732820631, 1.175544413473833, 1.1474083142393565, 1.1199112790671786, 1.092909618037032, 1.0755929927697745, 1.0512861461262244, 1.0312571117882559, 1.0194388253067554, 1.0019061415077835, 0.980344379776871, 0.9647961249933756, 0.9493075664595542, 0.9344817410273759, 0.9198929201794295, 0.9079992279088673, 0.8952920956888009, 0.8869530532606651, 0.8771902552778059, 0.8683889858357474, 0.8579407806493952, 0.8482823463732029, 0.8408658204630546, 0.8330843095877946, 0.8243382853592501, 0.818115422616924, 0.8135611224181157, 0.8090075029660142 ], "peak_layer": 1, "peak_normalized": 1.3739040544627474, "population_cv": 0.16161403917150835, "rest_mean": 0.00014210757001786045, "spike_contrast": 0.8576421569620596, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012187744287075474, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00019089931447524577, 0.00018474484386388212, 0.00017900968668982387, 0.00017368212866131216, 0.0001685509632807225, 0.00016333791427314281, 0.00015942849859129637, 0.00015560787869617343, 0.00015185608936008066, 0.00014945000293664634, 0.00014607264893129468, 0.0001432896824553609, 0.00014164757158141583, 0.0001392114645568654, 0.00013621553080156446, 0.0001340551534667611, 0.00013190307072363794, 0.00012984307250007987, 0.0001278160052606836, 0.00012616341700777411, 0.0001243978040292859, 0.00012323912233114243, 0.00012188261462142691, 0.00012065970804542303, 0.00011920796532649547, 0.00011786595860030502, 0.00011683545744745061, 0.00011575424287002534, 0.00011453901242930442, 0.00011367436673026532, 0.00011304156214464456, 0.00011240885214647278 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 0.00014552358334185556, "first_to_last_ratio": 1.262638843036328, "imbalance_abs_log_ratio": 0.2332038508028458, "last_quartile_mean": 0.00011525352965691127, "mean": 0.00012953498708156985, "normalized": [ 1.1676304712442283, 1.1525836984023852, 1.1391521450612143, 1.1288261090295841, 1.1167600441451297, 1.1054423860149858, 1.094731137320412, 1.082319740349084, 1.067957453000188, 1.0563935463552665, 1.04560972624621, 1.0337959762176818, 1.0235535208863231, 1.0159040893770892, 1.0050628637064216, 0.9950792389307415, 0.9865814183778941, 0.9786230528286792, 0.9704418054120744, 0.9608381133357455, 0.9508407831226554, 0.9429749854457838, 0.9346839821902475, 0.9262276149685831, 0.9170542960948607, 0.9059488475652709, 0.8984648923618161, 0.8908037775288885, 0.8831483920181569, 0.8778579808382188, 0.8744051093280881, 0.8703028022960919 ], "peak_layer": 1, "peak_normalized": 1.1676304712442283, "population_cv": 0.0896642712448568, "rest_mean": 0.00013110964479252557, "spike_contrast": 0.9231344927669946, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012103183544240892, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00015124899800866842, 0.00014929991448298097, 0.00014756005839444697, 0.00014622247545048594, 0.00014465949789155275, 0.00014319346519187093, 0.00014180598373059183, 0.00014019827358424664, 0.00013833785487804562, 0.00013683992438018322, 0.0001354430423816666, 0.00013391274842433631, 0.0001325859921053052, 0.00013159512309357524, 0.00013019080506637692, 0.00012889757636003196, 0.00012779681128449738, 0.00012676592450588942, 0.00012570616672746837, 0.00012446215259842575, 0.00012316714855842292, 0.00012214825255796313, 0.00012107427755836397, 0.00011997888213954866, 0.00011879061639774591, 0.00011735207226593047, 0.0001163826382253319, 0.0001153902558144182, 0.00011439861555118114, 0.00011371332220733166, 0.00011326605454087257, 0.0001127346622524783 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00029742828155576717, "first_to_last_ratio": 2.2781890121807993, "imbalance_abs_log_ratio": 0.8233808344454435, "last_quartile_mean": 0.00013055469935352448, "mean": 0.00019329176166138495, "normalized": [ 1.8891850069388627, 1.7801603926271052, 1.6773544463608039, 1.566500262900107, 1.476456430986117, 1.3754925511314788, 1.3084504582544147, 1.236424592515421, 1.172965600373877, 1.150871569148191, 1.0802294364746856, 1.0319209662753672, 1.0080063247818516, 0.9698985570693867, 0.9067931019504507, 0.8756163462587222, 0.8444435806568995, 0.8113939671049883, 0.7809677247391495, 0.7613442367276676, 0.7402582678385915, 0.7293562882295931, 0.7162426695419845, 0.7062419232587185, 0.6940340549797547, 0.6866100033591334, 0.6831688143117768, 0.6753320518823956, 0.6715919825156708, 0.6686410475337363, 0.6640239865195754, 0.6600233567535221 ], "peak_layer": 1, "peak_normalized": 1.8891850069388627, "population_cv": 0.357915029863326, "rest_mean": 0.00020341356920888993, "spike_contrast": 0.6815376252628051, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001386340009048581, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00036516389809548855, 0.0003440903383307159, 0.0003242187958676368, 0.0003027915954589844, 0.00028538686456158757, 0.00026587137836031616, 0.0002529126941226423, 0.00023899068764876574, 0.00022672458726447076, 0.00022245399304665625, 0.0002087994507746771, 0.00019946182146668434, 0.00019483931828290224, 0.00018747340072877705, 0.00017527563613839447, 0.00016924942610785365, 0.00016322398732881993, 0.00015683576930314302, 0.0001509546273155138, 0.00014716156874783337, 0.0001430858246749267, 0.0001409785618307069, 0.00013844380737282336, 0.00013651074550580233, 0.00013415106514003128, 0.00013271605712361634, 0.00013205090363044292, 0.00013053612201474607, 0.00012981319741811603, 0.00012924280599690974, 0.00012835036613978446, 0.00012757707736454904 ] } } }, "pre_attention_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "de71d4cf2d509a1063f6eac7f50c4885e9a058dee7e4b993ad40f5527f528d00", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.02802751213312149, 0.01413465291261673, 0.014225807972252369, 0.01433455292135477, 0.01440478302538395, 0.009703479707241058, 0.00977881159633398, 0.009858808480203152, 0.009950309991836548, 0.00752287870272994, 0.007571155205368996, 0.007693085819482803, 0.007788818795233965, 0.006353048607707024, 0.006389216054230928, 0.0064707002602517605, 0.00656962301582098, 0.005553483963012695, 0.0056249531917274, 0.005688921548426151, 0.005768724251538515, 0.004966245498508215, 0.005057469941675663, 0.005098348017781973, 0.005216081161051989, 0.0046758754178881645, 0.004727107938379049, 0.004777040798217058, 0.004837613552808762, 0.0044186655431985855, 0.004510571248829365, 0.0045849340967834 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.0001580166108396952, "first_to_last_ratio": 1.9713740353643698, "imbalance_abs_log_ratio": 0.6787307795042399, "last_quartile_mean": 8.015557068574708e-05, "mean": 0.00011843875881822896, "normalized": [ 0.8918679597819287, 1.5814819593588525, 1.5722239987443252, 1.3246067564285042, 1.1252650527356378, 1.512743589128775, 1.3788112428745087, 1.2863035124436846, 1.097095055843533, 1.468154353173571, 1.1523254606068964, 1.0839888571909224, 1.054518217028451, 1.146335080549274, 1.1665208527784754, 0.8931136212023528, 0.8203804970547371, 1.073741223277434, 0.919654742394771, 0.8223971330190447, 0.7711288002244273, 0.8470775332584999, 0.8298711629738247, 0.7662487463744663, 0.7210382462310316, 0.7071376059209309, 0.7189559392079661, 0.727980518763353, 0.6111916361855597, 0.6986829946379066, 0.6244678178312321, 0.6046898327751223 ], "peak_layer": 2, "peak_normalized": 1.5814819593588525, "population_cv": 0.2904715201464359, "rest_mean": 0.00012310891098523926, "spike_contrast": 0.7572151875143335, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.321993711637333e-05, "top_five_layers": [ 2, 3, 6, 10, 7 ], "top_five_spike_overlap": 0 }, "values": [ 0.00010563173418631777, 0.0001873087603598833, 0.00018621225899551064, 0.00015688478015363216, 0.0001332749961875379, 0.00017916747310664505, 0.00016330469225067645, 0.00015234819147735834, 0.00012993857671972364, 0.00017388637934345752, 0.0001364799973089248, 0.0001283862948184833, 0.00012489582877606153, 0.00013577050413005054, 0.00013816128193866462, 0.00010577926877886057, 9.71648478298448e-05, 0.00012717257777694613, 0.00010892276623053476, 9.74036956904456e-05, 9.133153798757121e-05, 0.0001003268116619438, 9.828891052166e-05, 9.07535504666157e-05, 8.539887494407594e-05, 8.375250035896897e-05, 8.515224908478558e-05, 8.622110908618197e-05, 7.238877878990024e-05, 8.275114669231698e-05, 7.396119326585904e-05, 7.161871326388791e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.0001664285200604354, "first_to_last_ratio": 1.9719337364651075, "imbalance_abs_log_ratio": 0.6790146534176558, "last_quartile_mean": 8.439863722742302e-05, "mean": 0.0001245867656507471, "normalized": [ 0.8984334461398115, 1.5927952483854981, 1.5658595485525297, 1.3198735865176374, 1.131213894321221, 1.5125315092450449, 1.3796810696180641, 1.286366052942558, 1.0960606282761096, 1.4536884557404346, 1.1543342746982923, 1.0797559630746076, 1.0497530570909623, 1.1457182987306291, 1.1551962694199982, 0.9023721663305979, 0.8238710779144349, 1.0632503993742801, 0.9211624832492573, 0.8274335806257342, 0.7739032412019898, 0.850990856837025, 0.8281317613098801, 0.7681943949939335, 0.7208251570035091, 0.7089546837555952, 0.7203162530736348, 0.7236174125342815, 0.6145740290254166, 0.6994991973458914, 0.6274381906330746, 0.6042038120380651 ], "peak_layer": 2, "peak_normalized": 1.5927952483854981, "population_cv": 0.2889665774838781, "rest_mean": 0.00012946851159169995, "spike_contrast": 0.7586812913967177, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.822533756960183e-05, "top_five_layers": [ 2, 3, 6, 10, 7 ], "top_five_spike_overlap": 0 }, "values": [ 0.00011193291720701382, 0.0001984412083402276, 0.0001950853766174987, 0.000164438781212084, 0.00014093428035266697, 0.00018844140868168324, 0.00017189000209327787, 0.00016026418597903103, 0.00013655464863404632, 0.00018111034296453, 0.00014381477376446128, 0.0001345233031315729, 0.00013078533811494708, 0.00014274133718572557, 0.00014392216689884663, 0.00011242362961638719, 0.00010264343291055411, 0.0001324669283349067, 0.00011476465442683548, 0.00010308727360097691, 9.641810174798593e-05, 0.00010602219845168293, 0.00010317425767425448, 9.570685506332666e-05, 8.980527491075918e-05, 8.832637104205787e-05, 8.974187221610919e-05, 9.015315299620852e-05, 7.656779052922502e-05, 8.714834257261828e-05, 7.817049481673166e-05, 7.527579873567447e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.14759047608822584, "first_to_last_ratio": 1.9719337947467823, "imbalance_abs_log_ratio": 0.6790146829732503, "last_quartile_mean": 0.07484555337578058, "mean": 0.1104847895912826, "normalized": [ 0.8984334974418968, 1.5927952138130326, 1.565859646322909, 1.3198735575362357, 1.1312139236939245, 1.5125315007911495, 1.3796810264894532, 1.286366010451999, 1.0960606168617568, 1.4536884922319049, 1.1543342013148132, 1.0797558953237962, 1.0497530983161607, 1.1457182562234747, 1.1551962948455465, 0.9023721940039697, 0.8238711061937815, 1.0632504861804335, 0.9211625151546727, 0.8274335809310764, 0.7739032642161865, 0.8509909070635955, 0.8281317391228538, 0.7681943896830757, 0.7208251689669645, 0.7089546589522648, 0.7203162343708721, 0.7236173971219663, 0.6145740294939214, 0.6994991437375198, 0.6274381319743458, 0.6042038211744466 ], "peak_layer": 2, "peak_normalized": 1.5927952138130326, "population_cv": 0.28896658018674404, "rest_mean": 0.11481397047086998, "spike_contrast": 0.7586813040631772, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.08710721284151077, "top_five_layers": [ 2, 3, 6, 10, 7 ], "top_five_spike_overlap": 0 }, "values": [ 0.09926323592662811, 0.1759796440601349, 0.1730036735534668, 0.14582595229148865, 0.12498193234205246, 0.16711172461509705, 0.15243376791477203, 0.14212387800216675, 0.12109802663326263, 0.16061046719551086, 0.1275363713502884, 0.1192966029047966, 0.11598175019025803, 0.12658444046974182, 0.12763161957263947, 0.09969840198755264, 0.0910252258181572, 0.11747300624847412, 0.10177444666624069, 0.09141882508993149, 0.08550453931093216, 0.09402155131101608, 0.09149596095085144, 0.08487379550933838, 0.0796402171254158, 0.0783287063241005, 0.07958398759365082, 0.07994871586561203, 0.06790108233690262, 0.07728401571512222, 0.069322369992733, 0.06675533205270767 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 5.093040158499207e-05, "first_to_last_ratio": 0.8860093039525244, "imbalance_abs_log_ratio": 0.12102782735710801, "last_quartile_mean": 5.748291960117058e-05, "mean": 5.7093861016710434e-05, "normalized": [ 0.4337136137178416, 0.9185569667519601, 0.9444095926066538, 0.8728787927927439, 0.7849140301431445, 1.1296292591692207, 1.0726593944235638, 0.9796135302182263, 0.9386642177229508, 1.3046353648934683, 1.0461832118506482, 1.0032917361864164, 0.9662905330123907, 1.0760080966276753, 1.1828739682986207, 0.9087955578115051, 0.9194941496879511, 1.198292373503509, 1.0946132296674007, 1.0673695485587213, 0.9244260207272295, 1.0525679458300632, 1.125653431968099, 0.9999504891362015, 0.9629027708994021, 1.0063234987710272, 0.9975933861195996, 1.0791525148711678, 0.9412468232151083, 1.085173475571988, 1.0031556955576197, 0.9789667796878816 ], "peak_layer": 10, "peak_normalized": 1.3046353648934683, "population_cv": 0.1441687540611157, "rest_mean": 5.695535414647397e-05, "spike_contrast": 1.0155638391297448, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.784179811598733e-05, "top_five_layers": [ 10, 18, 15, 6, 23 ], "top_five_spike_overlap": 1 }, "values": [ 2.4762384782661684e-05, 5.2443963795667514e-05, 5.3919990023132414e-05, 4.9836020480142906e-05, 4.481377254705876e-05, 6.449489592341706e-05, 6.124226638348773e-05, 5.592991874436848e-05, 5.3591964388033375e-05, 7.448667020071298e-05, 5.973063889541663e-05, 5.728179894504137e-05, 5.516925739357248e-05, 6.143345672171563e-05, 6.75348419463262e-05, 5.1886647270293906e-05, 5.249747118796222e-05, 6.84151382301934e-05, 6.249569560168311e-05, 6.094024865888059e-05, 5.277905074763112e-05, 6.009516800986603e-05, 6.426790059776977e-05, 5.7091034250333905e-05, 5.4975836974335834e-05, 5.74548939766828e-05, 5.695645813830197e-05, 6.161298369988799e-05, 5.373941530706361e-05, 6.19567435933277e-05, 5.727403186028823e-05, 5.58929932594765e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 5.801741053801379e-05, "first_to_last_ratio": 0.954195857595535, "imbalance_abs_log_ratio": 0.046886327142486896, "last_quartile_mean": 6.08024129178375e-05, "mean": 6.296396202287724e-05, "normalized": [ 0.4796364558374551, 0.9568353419130547, 0.9804638447804884, 0.8949545354119591, 0.8237377234766136, 1.1597846916175047, 1.085649359633547, 0.9904450076993521, 0.9595078978373608, 1.2941748206155441, 1.0680753760655664, 1.0231559828609242, 0.9904019047543832, 1.096490963646882, 1.1850351594596897, 0.9422215946487361, 0.9418631353047864, 1.1705024179981143, 1.0926353878857977, 1.0382275689133567, 0.9320833313178482, 1.0784583323883348, 1.0985080196909318, 0.9917906747078287, 0.9476388143731139, 0.9855066582268529, 0.978396290111904, 1.028407839812513, 0.8982055143685478, 1.0282804954549911, 0.9451124964564701, 0.9138123627295472 ], "peak_layer": 10, "peak_normalized": 1.2941748206155441, "population_cv": 0.13431233496292436, "rest_mean": 6.285090866084729e-05, "spike_contrast": 1.0115120295379672, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.357445017783903e-05, "top_five_layers": [ 10, 15, 18, 6, 23 ], "top_five_spike_overlap": 1 }, "values": [ 3.019981159013696e-05, 6.0246144130360335e-05, 6.173388828756288e-05, 5.634988337988034e-05, 5.186579073779285e-05, 7.302463927771896e-05, 6.835678505012766e-05, 6.236234185053036e-05, 6.041441884008236e-05, 8.148637425620109e-05, 6.725025741616264e-05, 6.442195444833487e-05, 6.235962791834027e-05, 6.903941539349034e-05, 7.461450877599418e-05, 5.9326004702597857e-05, 5.930343468207866e-05, 7.369946979451925e-05, 6.879665306769311e-05, 6.537092122016475e-05, 5.8687659475253895e-05, 6.790400948375463e-05, 6.916641723364592e-05, 6.244707037694752e-05, 5.966709431959316e-05, 6.205140380188823e-05, 6.160370685392991e-05, 6.475263216998428e-05, 5.655457789544016e-05, 6.474461406469345e-05, 5.950802733423188e-05, 5.753724690293893e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 1.7888660295284353e-05, "first_to_last_ratio": 0.41855462748958633, "imbalance_abs_log_ratio": 0.8709478659353102, "last_quartile_mean": 4.2739129185065394e-05, "mean": 3.1986895351110434e-05, "normalized": [ 0.2593955501951102, 0.4997829486650695, 0.48480994473028793, 0.4968760363401999, 0.5474125689219601, 0.7788406177150305, 0.7554473604850604, 0.6514322393717875, 0.7578135841604356, 0.9674810188795085, 0.9032772556875033, 0.9019220651946442, 0.8311356963419073, 0.9587970177975769, 1.1401394869948454, 0.8953886492354074, 1.0262272842074007, 1.1413916918814533, 1.174037958462901, 1.194927715786966, 1.0267140632187253, 1.3434898267078448, 1.3729067467361273, 1.2011929483563188, 1.1735959904166378, 1.3266745689353452, 1.2739952988364465, 1.4049535244931144, 1.2752889026856997, 1.4680979746684197, 1.4182292846928506, 1.3483241791974143 ], "peak_layer": 30, "peak_normalized": 1.4680979746684197, "population_cv": 0.3184955307507873, "rest_mean": 3.066251991657299e-05, "spike_contrast": 1.276428773763581, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.913852269761264e-05, "top_five_layers": [ 30, 31, 28, 23, 32 ], "top_five_spike_overlap": 1 }, "values": [ 8.297258318634704e-06, 1.5986504877218977e-05, 1.5507564967265353e-05, 1.589352177688852e-05, 1.7510028555989265e-05, 2.491269333404489e-05, 2.416441566310823e-05, 2.083729486912489e-05, 2.4240103812189773e-05, 3.094671410508454e-05, 2.889303505071439e-05, 2.8849686714238487e-05, 2.6585450541460887e-05, 3.066893987124786e-05, 3.6469522456172854e-05, 2.8640703021665104e-05, 3.282582474639639e-05, 3.6509576602838933e-05, 3.755382931558415e-05, 3.822202779701911e-05, 3.2841395295690745e-05, 4.2974068492185324e-05, 4.391502443468198e-05, 3.842243313556537e-05, 3.7539692129939795e-05, 4.243620060151443e-05, 4.0751154301688075e-05, 4.494010136113502e-05, 4.0792532672639936e-05, 4.695989628089592e-05, 4.5364751713350415e-05, 4.3128704419359565e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00022091567916504573, "first_to_last_ratio": 1.232986382535498, "imbalance_abs_log_ratio": 0.20943917994887448, "last_quartile_mean": 0.00017917122386279516, "mean": 0.0002198539991695725, "normalized": [ 0.4672441326905094, 1.0299890566632133, 1.08986441571188, 1.0084939820045402, 0.8814237539069011, 1.2693826047962031, 1.1886514905905403, 1.1035827467935222, 1.0473307057033197, 1.4274044616309502, 1.1442998336482333, 1.0911283607583675, 1.1094038041384326, 1.1946873974128753, 1.2511466100570947, 0.9923165311956169, 0.9239703707694655, 1.2473578196113966, 1.1181336030529359, 1.0160745414007064, 0.9397394354798955, 0.9905446518023159, 1.0121913654482901, 0.9359943959598221, 0.8839446941554127, 0.8510229499852975, 0.8648513533654478, 0.8786898836618169, 0.7455559493639876, 0.8399702482950041, 0.7392957942269532, 0.7163130557190521 ], "peak_layer": 10, "peak_normalized": 1.4274044616309502, "population_cv": 0.18952709251819305, "rest_mean": 0.0002217885959436427, "spike_contrast": 0.9441746799407325, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00020940717658959329, "top_five_layers": [ 10, 6, 15, 18, 14 ], "top_five_spike_overlap": 0 }, "values": [ 0.00010272549116052687, 0.00022644721320830286, 0.00023961105034686625, 0.00022172143508214504, 0.0001937845372594893, 0.0002790788421407342, 0.0002613297838252038, 0.00024262708029709756, 0.00023025984410196543, 0.000313820579322055, 0.00025157889467664063, 0.00023988893372006714, 0.00024390686303377151, 0.000262656802078709, 0.00027506958576850593, 0.0002181647578254342, 0.00020313858112785965, 0.00027423660503700376, 0.0002458261442370713, 0.00022338805138133466, 0.00020660547306761146, 0.00021777520305477083, 0.00022253431961871684, 0.00020578211115207523, 0.0001943387760547921, 0.00018710079893935472, 0.0001901410287246108, 0.00019318348495289683, 0.00016391345707233995, 0.00018467081827111542, 0.00016253713693004102, 0.00015748428995721042 ] } } }, "pre_mlp_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "2999a257265024350fd2665e2a8d452a03cd662193b4245d15290408ad29922d", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.014061377383768559, 0.014165290631353855, 0.014281988143920898, 0.014350161887705326, 0.00966146495193243, 0.009741888381540775, 0.009820535778999329, 0.00991273857653141, 0.0074927168898284435, 0.007546746172010899, 0.00766373984515667, 0.007762390188872814, 0.006325984839349985, 0.006375366821885109, 0.006445344537496567, 0.006555322092026472, 0.005531130358576775, 0.005611324217170477, 0.0056777093559503555, 0.00574630219489336, 0.004946626722812653, 0.0050392793491482735, 0.005084683652967215, 0.005203457549214363, 0.004667261149734259, 0.004717131145298481, 0.004768412560224533, 0.004826710093766451, 0.0044052014127373695, 0.004504534415900707, 0.004574235528707504, 0.004644096828997135 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 3.729895206561196e-05, "first_to_last_ratio": 1.2469721805352876, "imbalance_abs_log_ratio": 0.22071835733540474, "last_quartile_mean": 2.9911615229138988e-05, "mean": 3.340667609563752e-05, "normalized": [ 1.308752778077389, 1.2456464596610868, 1.0236956931411572, 0.9922285647386262, 1.26903181062375, 1.1170846373362797, 1.0287316533628064, 0.9469237624012169, 1.1300261751959062, 1.1132698789554223, 1.0126240729566007, 0.9379202574439188, 1.1130503370514597, 1.070713488608195, 0.9897642722064988, 0.922085688720865, 1.036831181462568, 0.9898105546019274, 0.9395130252451869, 0.9001967292772246, 0.9975952535130206, 0.9404117749145336, 0.9224862220158919, 0.8885787590271026, 0.9461670180110204, 0.8974372095121397, 0.8904691498559152, 0.8684437934704528, 0.9244147277585869, 0.9015689206268677, 0.8740901912628669, 0.8604359589635168 ], "peak_layer": 1, "peak_normalized": 1.308752778077389, "population_cv": 0.1164905978879517, "rest_mean": 3.378375206213807e-05, "spike_contrast": 0.9285666618330386, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.137046587653458e-05, "top_five_layers": [ 1, 5, 2, 9, 6 ], "top_five_spike_overlap": 0 }, "values": [ 4.3721080146497115e-05, 4.161290780757554e-05, 3.419827044126578e-05, 3.314705827506259e-05, 4.239413465256803e-05, 3.73180846509058e-05, 3.4366505133220926e-05, 3.1633575417799875e-05, 3.7750418414361775e-05, 3.719064625329338e-05, 3.3828404411906376e-05, 3.1332798243965954e-05, 3.718331208801828e-05, 3.5768978705164045e-05, 3.306473445263691e-05, 3.080381793552078e-05, 3.463708344497718e-05, 3.3066280593629926e-05, 3.138600732199848e-05, 3.007258055731654e-05, 3.332634150865488e-05, 3.1416031561093405e-05, 3.0817198421573266e-05, 2.9684462788281962e-05, 3.1608295103069395e-05, 2.9980394174344838e-05, 2.9747614462394267e-05, 2.9011820515734144e-05, 3.0881623388268054e-05, 3.01184209092753e-05, 2.9200447897892445e-05, 2.874430538213346e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 6.705871965095866e-05, "first_to_last_ratio": 0.7180297289018083, "imbalance_abs_log_ratio": 0.33124430563536084, "last_quartile_mean": 9.33926785364747e-05, "mean": 8.228966771639534e-05, "normalized": [ 0.8685826408392191, 0.8249006353131848, 0.7154905339803773, 0.6884730939760333, 0.9470487095509537, 0.8638069620260335, 0.8256155016985178, 0.7853664463613442, 0.9851686390305121, 0.9692511243547932, 0.9083804947955202, 0.8729855457896263, 1.0422548534804916, 1.0255492616133721, 0.9871528463964067, 0.9398223255303129, 1.0958375595009802, 1.0726325633807163, 1.0495259774381736, 1.0230838788771643, 1.1576441025653552, 1.1142532603054807, 1.1008567438213033, 1.056908774400763, 1.159063578749664, 1.1192434432436902, 1.1032241584740294, 1.1060435702772593, 1.1786235078649367, 1.1628365878275773, 1.141840825041133, 1.1085318534950752 ], "peak_layer": 29, "peak_normalized": 1.1786235078649367, "population_cv": 0.1341844256762087, "rest_mean": 8.049536754023629e-05, "spike_contrast": 1.1426606459269575, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.197888866765424e-05, "top_five_layers": [ 29, 30, 25, 21, 31 ], "top_five_spike_overlap": 2 }, "values": [ 7.14753768988885e-05, 6.788079917896539e-05, 5.887747829547152e-05, 5.66542221349664e-05, 7.793232362018898e-05, 7.108238787623122e-05, 6.793962529627606e-05, 6.462754390668124e-05, 8.106919995043427e-05, 7.975935295689851e-05, 7.475032907677814e-05, 7.183769048424438e-05, 8.576680556870997e-05, 8.4392107964959e-05, 8.123247971525416e-05, 7.733766688033938e-05, 9.017610864248127e-05, 8.826657722238451e-05, 8.636514394311234e-05, 8.418923243880272e-05, 9.526214853394777e-05, 9.169153054244816e-05, 9.0589135652408e-05, 8.697267185198143e-05, 9.537895675748587e-05, 9.210217103827745e-05, 9.078394941752777e-05, 9.101595787797123e-05, 9.698853682493791e-05, 9.56894364207983e-05, 9.396170207764953e-05, 9.122071787714958e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.05946834245696664, "first_to_last_ratio": 0.7180297222365345, "imbalance_abs_log_ratio": 0.33124431491808765, "last_quartile_mean": 0.08282156102359295, "mean": 0.07297530013602227, "normalized": [ 0.8685826122359729, 0.8249005917343738, 0.71549052911611, 0.6884731350635278, 0.9470486699630107, 0.8638070113577097, 0.8256154769796886, 0.7853664269052657, 0.9851686341386658, 0.9692511556795396, 0.9083805454214465, 0.8729855579949275, 1.0422548040385637, 1.0255492365750694, 0.9871528950273681, 0.9398223256645052, 1.0958375053304956, 1.07263252629438, 1.0495259690486791, 1.0230837911446407, 1.1576441435704576, 1.1142533045483842, 1.1008568124470575, 1.0569088284972943, 1.1590636022344982, 1.119243411839159, 1.1032241423789504, 1.1060435591240758, 1.1786235037173105, 1.1628366076958818, 1.141840809884862, 1.1085318743481278 ], "peak_layer": 29, "peak_normalized": 1.1786235037173105, "population_cv": 0.13418442668798974, "rest_mean": 0.07138409617322462, "spike_contrast": 1.1426607032635474, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.08156780153512955, "top_five_layers": [ 29, 30, 25, 21, 31 ], "top_five_spike_overlap": 2 }, "values": [ 0.06338507682085037, 0.0601973682641983, 0.05221313610672951, 0.050241533666849136, 0.0691111609339714, 0.06303657591342926, 0.060249537229537964, 0.05731235072016716, 0.07189297676086426, 0.07073139399290085, 0.06628934293985367, 0.06370638310909271, 0.07605885714292526, 0.0748397633433342, 0.07203777879476547, 0.06858381628990173, 0.07996907085180283, 0.07827568054199219, 0.07658947259187698, 0.07465984672307968, 0.0844794288277626, 0.08131296932697296, 0.0803353562951088, 0.07712823897600174, 0.08458301424980164, 0.08167712390422821, 0.08050811290740967, 0.08071386069059372, 0.0860104039311409, 0.08485835045576096, 0.08332617580890656, 0.08089544624090195 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 7.408341275549901e-06, "first_to_last_ratio": 0.49124437919492936, "imbalance_abs_log_ratio": 0.7108135577065074, "last_quartile_mean": 1.5080765479069669e-05, "mean": 1.1844782648040564e-05, "normalized": [ 0.5657990507718705, 0.5382233871697658, 0.49942145823009426, 0.48906984415931193, 0.8062921756393574, 0.7122953827323945, 0.7013508394120195, 0.6911626993599347, 0.8940675881372303, 0.868722127857376, 0.9735911331661544, 0.854855860776803, 0.989483587379368, 1.0059088486455374, 1.0303729006301217, 0.9941025538400318, 1.1620912657632405, 1.127012997009919, 1.113401270263767, 1.066097770938846, 1.1826034542812638, 1.1643547935214302, 1.2049778184223279, 1.1791491538213539, 1.2365172470752577, 1.219921447624247, 1.2337761202582145, 1.2288278254433027, 1.3039217543779833, 1.3631114810840155, 1.2856635723507386, 1.3138525898567213 ], "peak_layer": 30, "peak_normalized": 1.3631114810840155, "population_cv": 0.2549101540724733, "rest_mean": 1.1420299651271543e-05, "spike_contrast": 1.237882653019464, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.4136990830593277e-05, "top_five_layers": [ 30, 32, 29, 31, 25 ], "top_five_spike_overlap": 1 }, "values": [ 6.701766778860474e-06, 6.3751390371180605e-06, 5.915538622502936e-06, 5.7929260037781205e-06, 9.550355571263935e-06, 8.436983989668079e-06, 8.307348252856173e-06, 8.186671948351432e-06, 1.0590036254143342e-05, 1.0289824786013924e-05, 1.1531975360412616e-05, 1.0125581866304856e-05, 1.1720218026312068e-05, 1.1914771675947122e-05, 1.220454305439489e-05, 1.1774928680097219e-05, 1.3764718460151926e-05, 1.334922399109928e-05, 1.3187996046326589e-05, 1.2627696378331166e-05, 1.4007680874783546e-05, 1.379152945446549e-05, 1.4272700354922563e-05, 1.3966765436634887e-05, 1.4646278032159898e-05, 1.4449704394792207e-05, 1.4613809980801307e-05, 1.4555198504240252e-05, 1.5444669770658948e-05, 1.614575921848882e-05, 1.5228405572997872e-05, 1.5562298358418047e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 5.6029601637419546e-05, "first_to_last_ratio": 0.6070213107493427, "imbalance_abs_log_ratio": 0.49919138022039955, "last_quartile_mean": 9.230252817360451e-05, "mean": 7.840498903988191e-05, "normalized": [ 0.6598133090619259, 0.6419677920829977, 0.6005256373772865, 0.5913463573822231, 0.8562376513989928, 0.8062968559250673, 0.7923156566726156, 0.768439227921366, 0.9790122729268586, 0.9665730330981341, 0.9163710065350301, 0.8881686280668529, 1.0631190324760253, 1.0511030519715439, 1.0188080221730096, 0.9712812246823179, 1.1361272291219966, 1.1138561419734825, 1.0918134348286874, 1.0639217496772315, 1.203414696914147, 1.1592426077267617, 1.1442664103435203, 1.097952968622282, 1.2042281788778202, 1.1625445130141736, 1.1452986211445626, 1.1473932954411665, 1.2233942816523278, 1.205318760675343, 1.1829469858778916, 1.146901364356358 ], "peak_layer": 29, "peak_normalized": 1.2233942816523278, "population_cv": 0.1898597249088972, "rest_mean": 7.605543874820728e-05, "spike_contrast": 1.1977126437531995, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.109256061492487e-05, "top_five_layers": [ 29, 30, 25, 21, 31 ], "top_five_spike_overlap": 2 }, "values": [ 5.173265526536852e-05, 5.033347770222463e-05, 4.708420601673424e-05, 4.636450466932729e-05, 6.713330367347226e-05, 6.321769615169615e-05, 6.212150037754327e-05, 6.024946924299002e-05, 7.675944652874023e-05, 7.578414806630462e-05, 7.184805872384459e-05, 6.963685154914856e-05, 8.335383608937263e-05, 8.241172326961532e-05, 7.987963181221858e-05, 7.615329377586022e-05, 8.907804294722155e-05, 8.733187860343605e-05, 8.560362039133906e-05, 8.341677312273532e-05, 9.435371612198651e-05, 9.089040395338088e-05, 8.971619536168873e-05, 8.608499047113582e-05, 9.441749716643244e-05, 9.114928980125114e-05, 8.97971258382313e-05, 8.996135875349864e-05, 9.592021524440497e-05, 9.450300422031432e-05, 9.274894546251744e-05, 8.992278890218586e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 4.8511469685763586e-05, "first_to_last_ratio": 0.5271352332510719, "imbalance_abs_log_ratio": 0.6402981537748061, "last_quartile_mean": 9.202850924339145e-05, "mean": 7.532146378252946e-05, "normalized": [ 0.548104082267993, 0.5416830199005969, 0.528884702689083, 0.5247448675895723, 0.7777813723908718, 0.7487613756360456, 0.7496059866168319, 0.7329069161583202, 0.9480676391902623, 0.9399123881532756, 0.9000936195631334, 0.883441930780847, 1.0560589316963704, 1.0534552095740957, 1.032095781768693, 0.9835620691098637, 1.1601754560133901, 1.1467788545782247, 1.1233340534265408, 1.0967375255670042, 1.2396083970871608, 1.1951467072205837, 1.1828066060767042, 1.1317734013234224, 1.244698956870482, 1.2025862582796845, 1.1881970083126314, 1.1882256015362047, 1.2689460104605979, 1.2589162611011824, 1.2307315535498626, 1.1921774555104672 ], "peak_layer": 29, "peak_normalized": 1.2689460104605979, "population_cv": 0.23486334953575583, "rest_mean": 7.254842300123225e-05, "spike_contrast": 1.244629176846487, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.029588400153444e-05, "top_five_layers": [ 29, 30, 25, 21, 31 ], "top_five_spike_overlap": 2 }, "values": [ 4.1284001781605184e-05, 4.080035796505399e-05, 3.9836369978729635e-05, 3.952455153921619e-05, 5.858363147126511e-05, 5.639780283672735e-05, 5.6461420172126964e-05, 5.520362174138427e-05, 7.140984234865755e-05, 7.079557690303773e-05, 6.77963689668104e-05, 6.654213939327747e-05, 7.954390457598493e-05, 7.934778841445222e-05, 7.773896504659206e-05, 7.408333476632833e-05, 8.738611359149218e-05, 8.637706196168438e-05, 8.461116522084922e-05, 8.26078758109361e-05, 9.336911898571998e-05, 9.002019942272455e-05, 8.909072494134307e-05, 8.524682925781235e-05, 9.375254740007222e-05, 9.058055729838088e-05, 8.949673792812973e-05, 8.949889161158353e-05, 9.557887096889317e-05, 9.482341556577012e-05, 9.270050213672221e-05, 8.979655103757977e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 9.61669284151867e-05, "first_to_last_ratio": 0.8297833624837898, "imbalance_abs_log_ratio": 0.18659062131596024, "last_quartile_mean": 0.00011589401856326731, "mean": 0.00010716335873439675, "normalized": [ 0.9380742209455786, 0.8898820858744699, 0.7837417114644826, 0.7583102664636657, 1.0822070619439073, 0.9520749081713696, 0.9107982467806989, 0.8640017807748708, 1.0669538303069324, 1.0266950649381863, 0.9594029855154438, 0.8901506143407661, 1.0795289063552718, 1.0379165674077444, 0.9706971366471856, 0.914979822300863, 1.052939971267624, 1.0225707935964177, 0.9851238030379684, 0.9636652893153618, 1.0990044510799908, 1.0523904893769196, 1.0422234788441622, 1.0049019599938585, 1.0962247910521359, 1.0599378706264646, 1.05039930496957, 1.0583562345651827, 1.1199788667078585, 1.1100529546792268, 1.0850973923636869, 1.0717171382921347 ], "peak_layer": 29, "peak_normalized": 1.1199788667078585, "population_cv": 0.09156717959150794, "rest_mean": 0.00010599351123716958, "spike_contrast": 1.0706366257223145, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00011348053521942347, "top_five_layers": [ 29, 30, 21, 25, 31 ], "top_five_spike_overlap": 2 }, "values": [ 0.00010052718425868079, 9.536275319987908e-05, 8.398839418077841e-05, 8.12630751170218e-05, 0.00011597294360399246, 0.00010202754492638633, 9.760419925441965e-05, 9.25893327803351e-05, 0.00011433835607022047, 0.00011002409155480564, 0.00010281284630764276, 9.539152961224318e-05, 0.000115685943455901, 0.0001112266254494898, 0.0001040231654769741, 9.805231093196198e-05, 0.00011283658386673778, 0.00010958212078548968, 0.00010556917550275102, 0.00010326960909878835, 0.00011777300824178383, 0.0001127776995417662, 0.00011168816854478791, 0.00010768866923172027, 0.00011747513053705916, 0.00011358650226611644, 0.00011256431753281504, 0.00011341700883349404, 0.00012002069706795737, 0.00011895700299646705, 0.00011628268111962825, 0.00011484880815260112 ] } } } }, "selector": { "expected_uniform_count": 64, "identities_unique": true, "identity_order_sha256": "97fd9721a263024f6a746638f32e0ad9a988cd4c111c00e9e178c1022e5c9d5e", "passed": true, "uniform_count": 64, "uniform_indices": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63 ], "visit_count": 65, "visits": [ { "branch": "attention", "coefficient": "uniform", "group": 1, "index": 0, "kind": "depth", "layer": 1, "offset": 1, "sources": 1, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 1, "index": 1, "kind": "depth", "layer": 1, "offset": 1, "sources": 2, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 1, "index": 2, "kind": "depth", "layer": 2, "offset": 2, "sources": 2, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 1, "index": 3, "kind": "depth", "layer": 2, "offset": 2, "sources": 2, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 1, "index": 4, "kind": "depth", "layer": 3, "offset": 3, "sources": 2, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 1, "index": 5, "kind": "depth", "layer": 3, "offset": 3, "sources": 2, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 1, "index": 6, "kind": "depth", "layer": 4, "offset": 4, "sources": 2, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 1, "index": 7, "kind": "depth", "layer": 4, "offset": 4, "sources": 2, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 2, "index": 8, "kind": "depth", "layer": 5, "offset": 1, "sources": 2, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 2, "index": 9, "kind": "depth", "layer": 5, "offset": 1, "sources": 3, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 2, "index": 10, "kind": "depth", "layer": 6, "offset": 2, "sources": 3, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 2, "index": 11, "kind": "depth", "layer": 6, "offset": 2, "sources": 3, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 2, "index": 12, "kind": "depth", "layer": 7, "offset": 3, "sources": 3, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 2, "index": 13, "kind": "depth", "layer": 7, "offset": 3, "sources": 3, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 2, "index": 14, "kind": "depth", "layer": 8, "offset": 4, "sources": 3, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 2, "index": 15, "kind": "depth", "layer": 8, "offset": 4, "sources": 3, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 3, "index": 16, "kind": "depth", "layer": 9, "offset": 1, "sources": 3, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 3, "index": 17, "kind": "depth", "layer": 9, "offset": 1, "sources": 4, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 3, "index": 18, "kind": "depth", "layer": 10, "offset": 2, "sources": 4, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 3, "index": 19, "kind": "depth", "layer": 10, "offset": 2, "sources": 4, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 3, "index": 20, "kind": "depth", "layer": 11, "offset": 3, "sources": 4, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 3, "index": 21, "kind": "depth", "layer": 11, "offset": 3, "sources": 4, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 3, "index": 22, "kind": "depth", "layer": 12, "offset": 4, "sources": 4, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 3, "index": 23, "kind": "depth", "layer": 12, "offset": 4, "sources": 4, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 4, "index": 24, "kind": "depth", "layer": 13, "offset": 1, "sources": 4, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 4, "index": 25, "kind": "depth", "layer": 13, "offset": 1, "sources": 5, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 4, "index": 26, "kind": "depth", "layer": 14, "offset": 2, "sources": 5, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 4, "index": 27, "kind": "depth", "layer": 14, "offset": 2, "sources": 5, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 4, "index": 28, "kind": "depth", "layer": 15, "offset": 3, "sources": 5, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 4, "index": 29, "kind": "depth", "layer": 15, "offset": 3, "sources": 5, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 4, "index": 30, "kind": "depth", "layer": 16, "offset": 4, "sources": 5, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 4, "index": 31, "kind": "depth", "layer": 16, "offset": 4, "sources": 5, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 5, "index": 32, "kind": "depth", "layer": 17, "offset": 1, "sources": 5, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 5, "index": 33, "kind": "depth", "layer": 17, "offset": 1, "sources": 6, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 5, "index": 34, "kind": "depth", "layer": 18, "offset": 2, "sources": 6, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 5, "index": 35, "kind": "depth", "layer": 18, "offset": 2, "sources": 6, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 5, "index": 36, "kind": "depth", "layer": 19, "offset": 3, "sources": 6, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 5, "index": 37, "kind": "depth", "layer": 19, "offset": 3, "sources": 6, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 5, "index": 38, "kind": "depth", "layer": 20, "offset": 4, "sources": 6, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 5, "index": 39, "kind": "depth", "layer": 20, "offset": 4, "sources": 6, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 6, "index": 40, "kind": "depth", "layer": 21, "offset": 1, "sources": 6, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 6, "index": 41, "kind": "depth", "layer": 21, "offset": 1, "sources": 7, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 6, "index": 42, "kind": "depth", "layer": 22, "offset": 2, "sources": 7, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 6, "index": 43, "kind": "depth", "layer": 22, "offset": 2, "sources": 7, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 6, "index": 44, "kind": "depth", "layer": 23, "offset": 3, "sources": 7, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 6, "index": 45, "kind": "depth", "layer": 23, "offset": 3, "sources": 7, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 6, "index": 46, "kind": "depth", "layer": 24, "offset": 4, "sources": 7, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 6, "index": 47, "kind": "depth", "layer": 24, "offset": 4, "sources": 7, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 7, "index": 48, "kind": "depth", "layer": 25, "offset": 1, "sources": 7, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 7, "index": 49, "kind": "depth", "layer": 25, "offset": 1, "sources": 8, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 7, "index": 50, "kind": "depth", "layer": 26, "offset": 2, "sources": 8, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 7, "index": 51, "kind": "depth", "layer": 26, "offset": 2, "sources": 8, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 7, "index": 52, "kind": "depth", "layer": 27, "offset": 3, "sources": 8, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 7, "index": 53, "kind": "depth", "layer": 27, "offset": 3, "sources": 8, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 7, "index": 54, "kind": "depth", "layer": 28, "offset": 4, "sources": 8, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 7, "index": 55, "kind": "depth", "layer": 28, "offset": 4, "sources": 8, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 8, "index": 56, "kind": "depth", "layer": 29, "offset": 1, "sources": 8, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 8, "index": 57, "kind": "depth", "layer": 29, "offset": 1, "sources": 9, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 8, "index": 58, "kind": "depth", "layer": 30, "offset": 2, "sources": 9, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 8, "index": 59, "kind": "depth", "layer": 30, "offset": 2, "sources": 9, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 8, "index": 60, "kind": "depth", "layer": 31, "offset": 3, "sources": 9, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 8, "index": 61, "kind": "depth", "layer": 31, "offset": 3, "sources": 9, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 8, "index": 62, "kind": "depth", "layer": 32, "offset": 4, "sources": 9, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 8, "index": 63, "kind": "depth", "layer": 32, "offset": 4, "sources": 9, "use_uniform": true }, { "branch": "output", "coefficient": "detached_learned", "group": null, "index": 64, "kind": "output", "layer": null, "offset": null, "sources": 9, "use_uniform": false } ], "visits_sha256": "07670769be21c5fc9c1f1c3f868748e2c48b84de5b27352f602103abf0cbce67" } }, "uniform_except_group_6": { "bits_per_byte": 7.959341947457508, "forward": { "activation_sha256": { "attention_branch_output": "676759a2d7c0615ad7e1f246e90927fb829d00c8b6c3745032ae6b27ab7d902a", "mlp_branch_output": "afe29f3ef4d2992353f0bb59fe7c5c2c5050fca671a53184825a8d63cd83a2bc", "post_attention_state": "31853ffb07b24cc59e774bb20656840b4358b3080b42516d380f2dc0f46404c8", "post_mlp_state": "7fd9c911e93175896ad618842c120e77b3f017c17b7eec9c2bfb01fc6dbc8ac2", "pre_attention_input": "de71d4cf2d509a1063f6eac7f50c4885e9a058dee7e4b993ad40f5527f528d00", "pre_mlp_input": "2999a257265024350fd2665e2a8d452a03cd662193b4245d15290408ad29922d" }, "logits_sha256": "873a309a197bfd08fa523edcb440d2697903a34abc02dad1c7f5991509c3c151", "loss_nats": 5.516995429992676, "mixer_summary_sha256": "8450b24bdc777885a3357dc7b8080e7410acbe96943686433da76f379295133a" }, "loss_nats": 5.516995429992676, "loss_scale": 1.0, "mixers": [ { "branch": "attention", "entropy_mean": -0.0, "group": 1, "latest_source_mass": 1.0, "layer": 1, "max_source_mass": 1.0, "mixer_index": 0, "normalized_entropy": 1.0, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 1.0, "median": 1.0, "p05": 1.0, "p95": 1.0 } ], "sources": 1 }, { "branch": "mlp", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 1, "max_source_mass": 0.5, "mixer_index": 1, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 2, "max_source_mass": 0.5, "mixer_index": 2, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 2, "max_source_mass": 0.5, "mixer_index": 3, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 3, "max_source_mass": 0.5, "mixer_index": 4, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 3, "max_source_mass": 0.5, "mixer_index": 5, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 4, "max_source_mass": 0.5, "mixer_index": 6, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 4, "max_source_mass": 0.5, "mixer_index": 7, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6931471824645996, "group": 2, "latest_source_mass": 0.5, "layer": 5, "max_source_mass": 0.5, "mixer_index": 8, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "completed_group_1", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 5, "max_source_mass": 0.3333333432674408, "mixer_index": 9, "normalized_entropy": 1.0000000180537645, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 6, "max_source_mass": 0.3333333432674408, "mixer_index": 10, "normalized_entropy": 1.0000000180537645, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 6, "max_source_mass": 0.3333333432674408, "mixer_index": 11, "normalized_entropy": 1.0000000180537645, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 7, "max_source_mass": 0.3333333432674408, "mixer_index": 12, "normalized_entropy": 1.0000000180537645, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 7, "max_source_mass": 0.3333333432674408, "mixer_index": 13, "normalized_entropy": 1.0000000180537645, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 8, "max_source_mass": 0.3333333432674408, "mixer_index": 14, "normalized_entropy": 1.0000000180537645, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 8, "max_source_mass": 0.3333333432674408, "mixer_index": 15, "normalized_entropy": 1.0000000180537645, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0986123085021973, "group": 3, "latest_source_mass": 0.3333333432674408, "layer": 9, "max_source_mass": 0.3333333432674408, "mixer_index": 16, "normalized_entropy": 1.0000000180537645, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_2", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 9, "max_source_mass": 0.25, "mixer_index": 17, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 10, "max_source_mass": 0.25, "mixer_index": 18, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 10, "max_source_mass": 0.25, "mixer_index": 19, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 11, "max_source_mass": 0.25, "mixer_index": 20, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 11, "max_source_mass": 0.25, "mixer_index": 21, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 12, "max_source_mass": 0.25, "mixer_index": 22, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 12, "max_source_mass": 0.25, "mixer_index": 23, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3862943649291992, "group": 4, "latest_source_mass": 0.25, "layer": 13, "max_source_mass": 0.25, "mixer_index": 24, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_3", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 13, "max_source_mass": 0.20000000298023224, "mixer_index": 25, "normalized_entropy": 1.0000000186840277, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 14, "max_source_mass": 0.20000000298023224, "mixer_index": 26, "normalized_entropy": 1.0000000186840277, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 14, "max_source_mass": 0.20000000298023224, "mixer_index": 27, "normalized_entropy": 1.0000000186840277, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 15, "max_source_mass": 0.20000000298023224, "mixer_index": 28, "normalized_entropy": 1.0000000186840277, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 15, "max_source_mass": 0.20000000298023224, "mixer_index": 29, "normalized_entropy": 1.0000000186840277, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 16, "max_source_mass": 0.20000000298023224, "mixer_index": 30, "normalized_entropy": 1.0000000186840277, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 16, "max_source_mass": 0.20000000298023224, "mixer_index": 31, "normalized_entropy": 1.0000000186840277, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.6094379425048828, "group": 5, "latest_source_mass": 0.20000000298023224, "layer": 17, "max_source_mass": 0.20000000298023224, "mixer_index": 32, "normalized_entropy": 1.0000000186840277, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_4", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 17, "max_source_mass": 0.1666666716337204, "mixer_index": 33, "normalized_entropy": 1.0000001451965654, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 18, "max_source_mass": 0.1666666716337204, "mixer_index": 34, "normalized_entropy": 1.0000001451965654, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 18, "max_source_mass": 0.1666666716337204, "mixer_index": 35, "normalized_entropy": 1.0000001451965654, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 19, "max_source_mass": 0.1666666716337204, "mixer_index": 36, "normalized_entropy": 1.0000001451965654, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 19, "max_source_mass": 0.1666666716337204, "mixer_index": 37, "normalized_entropy": 1.0000001451965654, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 20, "max_source_mass": 0.1666666716337204, "mixer_index": 38, "normalized_entropy": 1.0000001451965654, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 20, "max_source_mass": 0.1666666716337204, "mixer_index": 39, "normalized_entropy": 1.0000001451965654, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.791759729385376, "group": 6, "latest_source_mass": 0.1666666716337204, "layer": 21, "max_source_mass": 0.1666666716337204, "mixer_index": 40, "normalized_entropy": 1.0000001451965654, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_5", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 21, "max_source_mass": 0.1428571492433548, "mixer_index": 41, "normalized_entropy": 1.00000003408302, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 22, "max_source_mass": 0.1428571492433548, "mixer_index": 42, "normalized_entropy": 1.00000003408302, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 22, "max_source_mass": 0.1428571492433548, "mixer_index": 43, "normalized_entropy": 1.00000003408302, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 23, "max_source_mass": 0.1428571492433548, "mixer_index": 44, "normalized_entropy": 1.00000003408302, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 23, "max_source_mass": 0.1428571492433548, "mixer_index": 45, "normalized_entropy": 1.00000003408302, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 24, "max_source_mass": 0.1428571492433548, "mixer_index": 46, "normalized_entropy": 1.00000003408302, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 24, "max_source_mass": 0.1428571492433548, "mixer_index": 47, "normalized_entropy": 1.00000003408302, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9459102153778076, "group": 7, "latest_source_mass": 0.1428571492433548, "layer": 25, "max_source_mass": 0.1428571492433548, "mixer_index": 48, "normalized_entropy": 1.00000003408302, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_6", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 25, "max_source_mass": 0.125, "mixer_index": 49, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 26, "max_source_mass": 0.125, "mixer_index": 50, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 26, "max_source_mass": 0.125, "mixer_index": 51, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 27, "max_source_mass": 0.125, "mixer_index": 52, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 27, "max_source_mass": 0.125, "mixer_index": 53, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 28, "max_source_mass": 0.125, "mixer_index": 54, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 28, "max_source_mass": 0.125, "mixer_index": 55, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.079441547393799, "group": 8, "latest_source_mass": 0.125, "layer": 29, "max_source_mass": 0.125, "mixer_index": 56, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_7", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 29, "max_source_mass": 0.1111111119389534, "mixer_index": 57, "normalized_entropy": 1.0000000180537645, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 30, "max_source_mass": 0.1111111119389534, "mixer_index": 58, "normalized_entropy": 1.0000000180537645, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 30, "max_source_mass": 0.1111111119389534, "mixer_index": 59, "normalized_entropy": 1.0000000180537645, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 31, "max_source_mass": 0.1111111119389534, "mixer_index": 60, "normalized_entropy": 1.0000000180537645, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 31, "max_source_mass": 0.1111111119389534, "mixer_index": 61, "normalized_entropy": 1.0000000180537645, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 32, "max_source_mass": 0.1111111119389534, "mixer_index": 62, "normalized_entropy": 1.0000000180537645, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 32, "max_source_mass": 0.1111111119389534, "mixer_index": 63, "normalized_entropy": 1.0000000180537645, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "output", "entropy_mean": 2.1972246170043945, "group": null, "latest_source_mass": 0.1111111119389534, "layer": null, "max_source_mass": 0.1111111119389534, "mixer_index": null, "normalized_entropy": 1.0000000180537645, "offset": null, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_8", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 } ], "mode": "uniform_except_group_6", "optimizer_state_after": "b82a349700eda58e2330a22ca2ee1f7fb1406efd47fd9aa306122b368a114192", "optimizer_state_before": "b82a349700eda58e2330a22ca2ee1f7fb1406efd47fd9aa306122b368a114192", "optimizer_state_unchanged": true, "positions": { "attention_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "676759a2d7c0615ad7e1f246e90927fb829d00c8b6c3745032ae6b27ab7d902a", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.0022265110164880753, 0.0021613112185150385, 0.0025601957459002733, 0.0025323182344436646, 0.0027253651060163975, 0.002904455177485943, 0.0029208383057266474, 0.00299624796025455, 0.0030937574338167906, 0.0033804215490818024, 0.0034370317589491606, 0.0038969130255281925, 0.004255065228790045, 0.004119740333408117, 0.003970506135374308, 0.004385598469525576, 0.0041999719105660915, 0.005232699681073427, 0.004664790350943804, 0.004898969549685717, 0.005131533369421959, 0.005069608334451914, 0.0051318020559847355, 0.0058372304774820805, 0.0059690922498703, 0.00554921617731452, 0.0058550527319312096, 0.006255472078919411, 0.006502739619463682, 0.006122888531535864, 0.007071305997669697, 0.006241494789719582 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.00015962477027642308, "first_to_last_ratio": 3.7699040072151497, "imbalance_abs_log_ratio": 1.3270495388586014, "last_quartile_mean": 4.234186599205714e-05, "mean": 8.617327489446325e-05, "normalized": [ 2.5428430303860874, 2.3222064178412967, 2.1011513502014654, 1.8911470095267005, 1.7004447966680725, 1.5353700031893112, 1.4229335846259188, 1.3028620492588128, 1.2049362956920768, 1.1705197150696323, 1.0773612761873523, 1.0133200843661747, 0.9906112963967714, 0.9272530070793641, 0.8550207003106389, 0.8185600560490035, 0.7703816559177782, 0.7173918665339755, 0.6760157258502337, 0.6514535263006792, 0.6184138115448249, 0.6053606047045522, 0.5838821473278404, 0.5697017460729616, 0.5457761369570343, 0.5276086337508958, 0.5104655265174319, 0.4951752820428687, 0.47527333481689216, 0.465789239220051, 0.45896414036514044, 0.45180594922816086 ], "peak_layer": 1, "peak_normalized": 2.5428430303860874, "population_cv": 0.5705341151299013, "rest_mean": 9.280180475562466e-05, "spike_contrast": 0.5428688997682254, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.0379213644191625e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00021912511147093028, 0.00020011213200632483, 0.00018106309289578348, 0.00016296633111778647, 0.00014653289690613747, 0.00013230786134954542, 0.0001226188469445333, 0.00011227188952034339, 0.0001038333066389896, 0.00010086751717608422, 9.283974941354245e-05, 8.732111018616706e-05, 8.536421955795959e-05, 7.990442827576771e-05, 7.367993384832516e-05, 7.053800072753802e-05, 6.63863102090545e-05, 6.182000652188435e-05, 5.825448897667229e-05, 5.6137883802875876e-05, 5.329074338078499e-05, 5.216590579948388e-05, 5.031503678765148e-05, 4.909306517220102e-05, 4.703131708083674e-05, 4.5465763832908124e-05, 4.3988486140733585e-05, 4.2670875700423494e-05, 4.095585973118432e-05, 4.013858415419236e-05, 3.955044303438626e-05, 3.893359826179221e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.00021442661636683624, "first_to_last_ratio": 1.843144421770586, "imbalance_abs_log_ratio": 0.6114730379455638, "last_quartile_mean": 0.0001163373926829081, "mean": 0.00015221274247778638, "normalized": [ 1.7741446897870732, 1.648461275666186, 1.5301464224684427, 1.4217121808823485, 1.3265065496562456, 1.2454420544341354, 1.1897998831173078, 1.1336248235930768, 1.0869089599614004, 1.0643003125599004, 1.0227030055526172, 0.9941765648180964, 0.9788958472869692, 0.9511102301475627, 0.9228060651723192, 0.9052418824754537, 0.8846484396889256, 0.865292475366091, 0.8497311645646503, 0.8388691935254817, 0.8256303537004397, 0.8176325372729673, 0.8076852907347736, 0.8000670639426607, 0.7892588692968658, 0.7814415503461742, 0.7731202151361662, 0.7664994567889658, 0.758490072461763, 0.7529625284402992, 0.7485174912671357, 0.7441725498875049 ], "peak_layer": 1, "peak_normalized": 1.7741446897870732, "population_cv": 0.27364872432853743, "rest_mean": 0.00015762320577373935, "spike_contrast": 0.7803181014868814, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012299624067964033, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00027004742878489196, 0.0002509168116375804, 0.0002329077833564952, 0.00021640271006617695, 0.00020191119983792305, 0.00018957215070258826, 0.0001811027032090351, 0.00017255214333999902, 0.00016544139361940324, 0.00016200006939470768, 0.00015566842921543866, 0.0001513263414381072, 0.00014900042151566595, 0.0001447710965294391, 0.00014046284195501357, 0.0001377893495373428, 0.00013465476513374597, 0.00013170854072086513, 0.00012933991092722863, 0.0001276865805266425, 0.00012567146040964872, 0.00012445409083738923, 0.00012293999316170812, 0.00012178040196886286, 0.00012013525702059269, 0.00011894536146428436, 0.00011767874821089208, 0.00011667098442558199, 0.00011545185407157987, 0.00011461049143690616, 0.00011393390013836324, 0.00011327254469506443 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.19015566632151604, "first_to_last_ratio": 1.8431444167877922, "imbalance_abs_log_ratio": 0.6114730352421437, "last_quartile_mean": 0.10316916275769472, "mean": 0.13498378219082952, "normalized": [ 1.7741447141792424, 1.6484611693507762, 1.5301464164964773, 1.421712099711509, 1.3265065258282553, 1.245442086656075, 1.1897999853143832, 1.1336247998774232, 1.0869090145314644, 1.0643003539468234, 1.0227030148964142, 0.9941765578539046, 0.978895844117059, 0.9511102293531465, 0.9228060470726628, 0.9052419251832852, 0.8846484748200404, 0.8652925213936268, 0.8497311905993837, 0.8388692019864461, 0.8256303029992552, 0.8176324960789627, 0.8076852727084025, 0.800067049482784, 0.789258877083647, 0.7814415614650243, 0.7731201949085273, 0.7664994759042474, 0.758490077799444, 0.7529625178867759, 0.7485174642151933, 0.7441725362993381 ], "peak_layer": 1, "peak_normalized": 1.7741447141792424, "population_cv": 0.2736487184871982, "rest_mean": 0.13978183573042904, "spike_contrast": 0.7803180757143806, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.10907429307699204, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.23948076367378235, 0.2225155234336853, 0.20654495060443878, 0.1919080764055252, 0.17905686795711517, 0.16811448335647583, 0.16060370206832886, 0.1530209630727768, 0.14671508967876434, 0.14366328716278076, 0.13804832100868225, 0.13419771194458008, 0.1321350634098053, 0.12838445603847504, 0.12456385046243668, 0.12219297885894775, 0.11941319704055786, 0.116800457239151, 0.11469992995262146, 0.11323373764753342, 0.11144670099020004, 0.11036712676286697, 0.10902441293001175, 0.10799607634544373, 0.10653714835643768, 0.10548193752765656, 0.10435868799686432, 0.1034649983048439, 0.10238385945558548, 0.10163772851228714, 0.10103771835565567, 0.10045122355222702 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 6.883365585963475e-05, "first_to_last_ratio": 2.2497546817252747, "imbalance_abs_log_ratio": 0.8108211799277664, "last_quartile_mean": 3.059607183786284e-05, "mean": 4.691841348858361e-05, "normalized": [ 1.7359303646705315, 1.6475859007944922, 1.5536416135247637, 1.4921074478374423, 1.436565604934822, 1.343834173349448, 1.2787439414688093, 1.2483315192759759, 1.191227587726315, 1.1603086071531024, 1.0995306735006558, 1.059977946939621, 1.0583984121521266, 1.0176934649371145, 0.9754909410255406, 0.9631621800279858, 0.9242301483750994, 0.8752327792179452, 0.8454160845678106, 0.8232909667825862, 0.7863453139125789, 0.77746685672799, 0.7497386589361914, 0.7388508716017261, 0.7093180450413769, 0.6908354487677191, 0.6685471901846501, 0.6592354484095886, 0.631440334976849, 0.6235901145414114, 0.6199506557967245, 0.6139807028410055 ], "peak_layer": 1, "peak_normalized": 1.7359303646705315, "population_cv": 0.3220412433027962, "rest_mean": 4.907019663530971e-05, "spike_contrast": 0.7193528234378936, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.529878449626267e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 8.144709863699973e-05, 7.730211655143648e-05, 7.289439963642508e-05, 7.000731420703232e-05, 6.740137905580923e-05, 6.305056740529835e-05, 5.999663699185476e-05, 5.856973439222202e-05, 5.5890508519951254e-05, 5.4439839004771784e-05, 5.1588234782684594e-05, 4.9732483603293076e-05, 4.965837433701381e-05, 4.774856279254891e-05, 4.576848732540384e-05, 4.519004141911864e-05, 4.3363412260077894e-05, 4.106453343410976e-05, 3.966558142565191e-05, 3.862750600092113e-05, 3.6894074582960457e-05, 3.647751145763323e-05, 3.517654840834439e-05, 3.4665710700210184e-05, 3.328007733216509e-05, 3.241290323785506e-05, 3.136717350571416e-05, 3.0930281354812905e-05, 2.9626178729813546e-05, 2.925785884144716e-05, 2.9087101211189292e-05, 2.8807000489905477e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 0.0001734125216898974, "first_to_last_ratio": 1.495517223041696, "imbalance_abs_log_ratio": 0.4024721155990803, "last_quartile_mean": 0.00011595488103921525, "mean": 0.00013990532806928968, "normalized": [ 1.3768889945483427, 1.3340940553978193, 1.2910049686488183, 1.2527203273458163, 1.2147536525239127, 1.1769854337196524, 1.14744939519443, 1.1220955988146075, 1.0944022472959734, 1.0773022658134643, 1.0508559212107247, 1.03206209473486, 1.0208171898483585, 1.000561882500425, 0.9805161605170362, 0.9663129296065233, 0.9487424991891081, 0.931931984738623, 0.9181974345490227, 0.9075104531838204, 0.8943640941785878, 0.8862664021756573, 0.8757830774574916, 0.86790401991425, 0.8562894022841879, 0.8478155487572495, 0.8387227683262297, 0.8314011142413483, 0.8224071457687988, 0.8161875050523302, 0.8112199678685613, 0.8064334645939688 ], "peak_layer": 1, "peak_normalized": 1.3768889945483427, "population_cv": 0.16325957486062354, "rest_mean": 0.0001431148236776978, "spike_contrast": 0.8564734849544942, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001225740517838858, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0001926341064972803, 0.00018664686649572104, 0.00018061847367789596, 0.00017526224837638438, 0.0001699505082797259, 0.00016466653323732316, 0.00016053428407758474, 0.00015698715287726372, 0.00015311270544771105, 0.00015072032692842185, 0.00014702034241054207, 0.00014439098595175892, 0.00014281776384450495, 0.00013998393842484802, 0.00013717943511437625, 0.00013519232743419707, 0.00013273413060232997, 0.00013038225006312132, 0.00012846071331296116, 0.00012696554767899215, 0.0001251263020094484, 0.00012399339175317436, 0.0001225267187692225, 0.00012142439663875848, 0.00011979944974882528, 0.00011861391249112785, 0.00011734178406186402, 0.00011631744564510882, 0.00011505914153531194, 0.00011418898066040128, 0.00011349399574100971, 0.00011282433843007311 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 0.00014675117381557357, "first_to_last_ratio": 1.2676837465168667, "imbalance_abs_log_ratio": 0.23719141364257146, "last_quartile_mean": 0.00011576323686313117, "mean": 0.00013042020850662084, "normalized": [ 1.169988069421413, 1.156175152925158, 1.1417665260225645, 1.1306726331060184, 1.1183621029407895, 1.1062694829504214, 1.0940115056230622, 1.0844989956600726, 1.0711591662817939, 1.0579765491085218, 1.0443900226505625, 1.035471661013805, 1.0260498632741444, 1.013992390086173, 1.0062479316494695, 0.9977001177294099, 0.9863163473759529, 0.9769959732660659, 0.9687585669786032, 0.9602628595878113, 0.9503845997255937, 0.9426104617668664, 0.9330243108845692, 0.9259760938667744, 0.914797625470175, 0.906289421438531, 0.8956357003805738, 0.888470885240542, 0.8806216558595509, 0.8756991509238615, 0.8712858841028995, 0.8681382926882502 ], "peak_layer": 1, "peak_normalized": 1.169988069421413, "population_cv": 0.09123006364857297, "rest_mean": 0.00013202972385678786, "spike_contrast": 0.9219804606102014, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012172882561571896, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00015259008796419948, 0.00015078860451467335, 0.00014890942838974297, 0.0001474625605624169, 0.00014585701865144074, 0.00014427989663090557, 0.00014268120867200196, 0.00014144058513920754, 0.00013970080181024969, 0.0001379815221298486, 0.00013620956451632082, 0.00013504642993211746, 0.0001338176371064037, 0.00013224509893916547, 0.00013123506505507976, 0.0001301202573813498, 0.00012863558367826045, 0.00012742001854348928, 0.00012634569429792464, 0.00012523768236860633, 0.00012394935765769333, 0.00012293545296415687, 0.00012168522516731173, 0.00012076599523425102, 0.00011930809705518186, 0.00011819845531135798, 0.00011680899478960782, 0.00011587455810513347, 0.00011485085997264832, 0.00011420886585256085, 0.00011363328667357564, 0.00011322277714498341 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00030002388848515693, "first_to_last_ratio": 2.2836575149168885, "imbalance_abs_log_ratio": 0.8257783308640597, "last_quartile_mean": 0.0001313786706305109, "mean": 0.0001947244400071213, "normalized": [ 1.8998833386658438, 1.7835645331795598, 1.6713073472446418, 1.5703972875779748, 1.4780250735507643, 1.3761669788219328, 1.3066447563403325, 1.2401013786965793, 1.1762106459027857, 1.150547637187856, 1.0768735950666006, 1.0264900078881174, 1.0140784159549234, 0.9666871251601796, 0.9096841069789114, 0.8754807117769676, 0.8421876125778792, 0.8073809902734768, 0.77911667677541, 0.7607651063686129, 0.7403822030704592, 0.7308855611321243, 0.7152791523187013, 0.7043381880055872, 0.6944157299089508, 0.686864629891504, 0.680599049276497, 0.6764772714226681, 0.6702412094778135, 0.6672258960044359, 0.6635863560708457, 0.6581114274310631 ], "peak_layer": 1, "peak_normalized": 1.8998833386658438, "population_cv": 0.3592853832280775, "rest_mean": 0.00020492727344390005, "spike_contrast": 0.6813594750078023, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013962913944851608, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0003699537192005664, 0.0003473036049399525, 0.0003254443872720003, 0.00030579473241232336, 0.00028780760476365685, 0.00026797334430739284, 0.0002544356684666127, 0.0002414780465187505, 0.0002290369593538344, 0.00022403974435292184, 0.00020969360775779933, 0.00019988269195891917, 0.0001974658516701311, 0.00018823760910890996, 0.00017713772831484675, 0.00017047749133780599, 0.00016399451124016196, 0.0001572168112033978, 0.00015171305858530104, 0.00014813955931458622, 0.00014417050988413393, 0.00014232128160074353, 0.00013928233238402754, 0.0001371518592350185, 0.00013521971413865685, 0.00013374933041632175, 0.00013252926873974502, 0.00013172665785532445, 0.00013051234418526292, 0.00012992518895771354, 0.00012921648158226162, 0.000128150379168801 ] } } }, "mlp_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "afe29f3ef4d2992353f0bb59fe7c5c2c5050fca671a53184825a8d63cd83a2bc", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.0027073738165199757, 0.002722575329244137, 0.0026850115973502398, 0.0026861224323511124, 0.00271838391199708, 0.0026645169127732515, 0.0026859452482312918, 0.0027359756641089916, 0.0026767742820084095, 0.0027451682835817337, 0.0026999011170119047, 0.002663193503394723, 0.002626648638397455, 0.0026531771291047335, 0.0026517712976783514, 0.002676903735846281, 0.002670086920261383, 0.0026355870068073273, 0.0026610493659973145, 0.0026388936676084995, 0.0026469926815479994, 0.00264862016774714, 0.002641494618728757, 0.002674198243767023, 0.002557571977376938, 0.00252733426168561, 0.0025468147359788418, 0.0026856842450797558, 0.002551089273765683, 0.0025778315030038357, 0.0026001501828432083, 0.0025662274565547705 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.00015858857113926206, "first_to_last_ratio": 3.757867690304466, "imbalance_abs_log_ratio": 1.3238516928952826, "last_quartile_mean": 4.220174423608114e-05, "mean": 8.573221236929385e-05, "normalized": [ 2.5461641896458365, 2.3032502265274117, 2.0924715218207024, 1.8886722264368014, 1.7031663195050915, 1.537434754996766, 1.4223955235407997, 1.3049501874408618, 1.2055297161045848, 1.1692542365736316, 1.0769571962730058, 1.0144505847411098, 0.9913601784485884, 0.9317001386730132, 0.857757082878185, 0.8155634736467603, 0.7682612945963089, 0.7217806418748633, 0.678338533281248, 0.6511820384364709, 0.6192882306865346, 0.6053793146733001, 0.5848721334223365, 0.5718144052517429, 0.5462281883927541, 0.527515083157495, 0.5112993585545178, 0.496920907994256, 0.47722358366421574, 0.46552107689700334, 0.4601234027970099, 0.45317424906679377 ], "peak_layer": 1, "peak_normalized": 2.5461641896458365, "population_cv": 0.5686091737198813, "rest_mean": 9.231269224848668e-05, "spike_contrast": 0.5437781067692293, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.0197621021652594e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00021828828903380781, 0.00019746273756027222, 0.00017939221288543195, 0.0001619200484128669, 0.00014601621660403907, 0.000131807682919316, 0.00012194511509733275, 0.00011187626660102978, 0.00010335272963857278, 0.00010024275252362713, 9.232992306351662e-05, 8.697109296917915e-05, 8.499150135321543e-05, 7.987671415321529e-05, 7.353741239057854e-05, 6.992006092332304e-05, 6.586474046343938e-05, 6.187985127326101e-05, 5.8155463193543255e-05, 5.582727681030519e-05, 5.309295011102222e-05, 5.1900507969548926e-05, 5.0142381951445714e-05, 4.902291402686387e-05, 4.682935104938224e-05, 4.522503513726406e-05, 4.3834825191879645e-05, 4.260212881490588e-05, 4.0913433622336015e-05, 3.991015182691626e-05, 3.944739728467539e-05, 3.885163096128963e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.00021242323236947414, "first_to_last_ratio": 1.8345651540958352, "imbalance_abs_log_ratio": 0.6068074801791611, "last_quartile_mean": 0.0001157894184871111, "mean": 0.00015111673860701558, "normalized": [ 1.771707915810251, 1.6366668487510905, 1.5242909365839221, 1.4186705636795862, 1.3264449439106374, 1.2453857912414694, 1.1893415085703798, 1.1330085309323608, 1.0858463851390703, 1.062861239127963, 1.023237806768965, 0.9942126959777651, 0.978436257513136, 0.9531027445378829, 0.9232414968561398, 0.904013237249231, 0.8853704564278231, 0.8680164030054869, 0.8515402788738432, 0.8395817783841374, 0.8267435189825344, 0.8185143642831039, 0.809172703595369, 0.8007918347797284, 0.7909896391756025, 0.7821052393452608, 0.7753388667989797, 0.7682453290925442, 0.7604995795745633, 0.7549901567959042, 0.7508572350862155, 0.7467737131490535 ], "peak_layer": 1, "peak_normalized": 1.771707915810251, "population_cv": 0.2714170004285523, "rest_mean": 0.00015645500981783562, "spike_contrast": 0.7816309251520459, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012229007406858728, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0002677347220014781, 0.00024732775636948645, 0.00023034587502479553, 0.00021438486874103546, 0.00020044803386554122, 0.0001881986390799284, 0.00017972940986510366, 0.00017121655400842428, 0.00016408956435043365, 0.00016061612404882908, 0.0001546283601783216, 0.00015024218009784818, 0.00014785809617023915, 0.0001440297783119604, 0.00013951724395155907, 0.000136611532070674, 0.0001337942958343774, 0.00013117180787958205, 0.0001286819897359237, 0.00012687486014328897, 0.00012493478425312787, 0.00012369122123345733, 0.00012227953993715346, 0.00012101305037504062, 0.00011953177454415709, 0.00011818919301731512, 0.00011716668086592108, 0.00011609472858253866, 0.00011492421617731452, 0.00011409165017539635, 0.00011346709652571008, 0.00011285000800853595 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.18837904930114746, "first_to_last_ratio": 1.8345651581276443, "imbalance_abs_log_ratio": 0.6068074823768531, "last_quartile_mean": 0.10268321540206671, "mean": 0.13401183625683188, "normalized": [ 1.7717078885231814, 1.6366668119471057, 1.524290957183596, 1.418670514273678, 1.326444920903592, 1.2453858718827995, 1.1893415500975306, 1.1330085716119305, 1.0858463365841577, 1.062861212873729, 1.0232377434953237, 0.9942127340740732, 0.9784362443707608, 0.9531027271661117, 0.9232414917376033, 0.904013241653558, 0.885370477646398, 0.8680164390463408, 0.8515402681847718, 0.8395816969955481, 0.8267435345782925, 0.8185144279720658, 0.8091727254708025, 0.8007918405918669, 0.7909896328038464, 0.7821052665924377, 0.7753388466443417, 0.7682453528580236, 0.7604996016741158, 0.7549901615676585, 0.7508572336421532, 0.7467736753526057 ], "peak_layer": 1, "peak_normalized": 1.7717078885231814, "population_cv": 0.27141699472708714, "rest_mean": 0.1387458680956452, "spike_contrast": 0.7816309473985974, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.10844806432724, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.23742982745170593, 0.2193327248096466, 0.2042730301618576, 0.19011864066123962, 0.1777593195438385, 0.16689644753932953, 0.15938584506511688, 0.151836559176445, 0.1455162614583969, 0.1424359828233719, 0.13712596893310547, 0.13323627412319183, 0.13112203776836395, 0.12772704660892487, 0.1237252876162529, 0.1211484745144844, 0.11865012347698212, 0.11632447689771652, 0.11411647498607635, 0.11251388490200043, 0.11079341918230057, 0.10969062149524689, 0.10843872278928757, 0.10731558501720428, 0.10600197315216064, 0.10481136292219162, 0.10390458256006241, 0.1029539704322815, 0.10191594809293747, 0.10117761790752411, 0.10062375664710999, 0.10007651150226593 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 6.873187476230669e-05, "first_to_last_ratio": 2.248757500145603, "imbalance_abs_log_ratio": 0.8103778415280068, "last_quartile_mean": 3.0564378221242805e-05, "mean": 4.685761763312257e-05, "normalized": [ 1.7400006587426948, 1.6405194725375354, 1.5551306973978087, 1.4944078341288562, 1.4327724019957897, 1.3414187773842097, 1.2804088050749736, 1.2499327782691996, 1.1927955171655855, 1.1608931065609767, 1.0981756972550076, 1.0577423904261343, 1.0542786038595535, 1.0218640087393278, 0.9775807389332813, 0.9570972402588515, 0.9209982861589985, 0.8815092249082567, 0.8478232134430647, 0.8204952153707635, 0.7871780668180614, 0.7759097745574741, 0.7506963524238334, 0.7421155343193898, 0.7072927325438048, 0.6891814936654451, 0.6695399899996352, 0.6616343983211914, 0.6345111343049281, 0.6212687927046295, 0.6200231913288072, 0.6148038704019305 ], "peak_layer": 1, "peak_normalized": 1.7400006587426948, "population_cv": 0.3217536161653951, "rest_mean": 4.900405669845727e-05, "spike_contrast": 0.719671983430414, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.526684668031521e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 8.153228554874659e-05, 7.687083416385576e-05, 7.286971958819777e-05, 7.002439087955281e-05, 6.71363013680093e-05, 6.285568815656006e-05, 5.999690620228648e-05, 5.8568872191244736e-05, 5.5891556257847697e-05, 5.439668530016206e-05, 5.145789691596292e-05, 4.9563288484932855e-05, 4.9400983698433265e-05, 4.7882112994557247e-05, 4.580710447044112e-05, 4.484729652176611e-05, 4.3155785533599555e-05, 4.130542220082134e-05, 3.972697595600039e-05, 3.844645107164979e-05, 3.688528886414133e-05, 3.635728353401646e-05, 3.517584264045581e-05, 3.477376594673842e-05, 3.314205241622403e-05, 3.229340290999971e-05, 3.1373048841487616e-05, 3.10026116494555e-05, 2.9731680115219206e-05, 2.9111175535945222e-05, 2.9052809622953646e-05, 2.8808244678657502e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 0.0001719075116852764, "first_to_last_ratio": 1.4894635014227988, "imbalance_abs_log_ratio": 0.39841598895878805, "last_quartile_mean": 0.00011541572621354135, "mean": 0.00013894635094402474, "normalized": [ 1.3738985779819257, 1.3296082759737424, 1.2883384004275333, 1.2499940655466777, 1.2130730176151365, 1.1755503979222366, 1.147409009170091, 1.11990595257071, 1.0929037170303153, 1.075598370036652, 1.0512762916941902, 1.0312594730951, 1.0194418974669357, 1.0018996501730997, 0.9803460168062212, 0.9648004557293595, 0.9493010807852901, 0.9344788929945884, 0.9198953859218769, 0.908001147597105, 0.8952913731815082, 0.886953676542665, 0.8771894463102521, 0.8683888934041172, 0.8579399877904575, 0.8482799643655919, 0.8408636339472488, 0.8330870307470335, 0.82434015220157, 0.8181187963300758, 0.8135579418994486, 0.809009026741244 ], "peak_layer": 1, "peak_normalized": 1.3738985779819257, "population_cv": 0.16161423472023242, "rest_mean": 0.0001421073115866252, "spike_contrast": 0.8576417505420824, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012187716347398237, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0001908981939777732, 0.0001847442181315273, 0.00017900991952046752, 0.00017368211410939693, 0.00016855206922627985, 0.00016333843814209104, 0.00015942829486448318, 0.00015560684551019222, 0.0001518549834145233, 0.00014945046859793365, 0.00014607100456487387, 0.0001432897406630218, 0.00014164773165248334, 0.00013921030040364712, 0.000136215501697734, 0.0001340555027127266, 0.0001319019211223349, 0.00012984243221580982, 0.0001278161071240902, 0.00012616344611160457, 0.00012439746933523566, 0.00012323897681199014, 0.00012188227265141904, 0.00012065946793882176, 0.00011920763063244522, 0.00011786540562752634, 0.00011683493357850239, 0.00011575440294109285, 0.0001145390560850501, 0.00011367462138878182, 0.00011304090730845928, 0.00011240885214647278 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 0.00014552221000485588, "first_to_last_ratio": 1.262697055715921, "imbalance_abs_log_ratio": 0.2332499537229619, "last_quartile_mean": 0.00011524712863320019, "mean": 0.00012953460168318998, "normalized": [ 1.1676213631634658, 1.1525055688027706, 1.1391580058081985, 1.1289307982409096, 1.1166731577821778, 1.1054001772932092, 1.094813144753513, 1.0822854389705017, 1.0679308603491235, 1.0564894822224282, 1.0456838360716472, 1.0337072702557388, 1.0234945545465333, 1.0158819477934218, 1.0051772952892828, 0.9951467950359763, 0.9866643397779161, 0.9787559418439278, 0.970448736962688, 0.9607908684400951, 0.9509939230158972, 0.9428061355388626, 0.934924980051271, 0.9261034265480621, 0.9170582603044295, 0.9058562786860701, 0.8981778407016658, 0.8907564927732574, 0.8831168120837269, 0.8778216107141017, 0.8745192645108896, 0.8703053916682415 ], "peak_layer": 1, "peak_normalized": 1.1676213631634658, "population_cv": 0.08967236486753827, "rest_mean": 0.00013110875065817877, "spike_contrast": 0.9231588022206524, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012103419721825049, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00015124736819416285, 0.00014928934979252517, 0.000147560378536582, 0.00014623560127802193, 0.00014464781270362437, 0.00014318757166620344, 0.00014181618462316692, 0.00014019341324456036, 0.0001383339986205101, 0.00013685194426216185, 0.00013545223919209093, 0.00013390085950959474, 0.00013257795944809914, 0.00013159186346456409, 0.00013020524056628346, 0.00012890594371128827, 0.00012780717224813998, 0.00012678276107180864, 0.0001257066905964166, 0.0001244556624442339, 0.00012318661902099848, 0.0001221260172314942, 0.00012110513489460573, 0.0001199624384753406, 0.00011879077646881342, 0.00011733973224181682, 0.00011634510883595794, 0.00011538378748809919, 0.0001143941844929941, 0.00011370827269274741, 0.00011328050459269434, 0.0001127346622524783 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00029741139769612346, "first_to_last_ratio": 2.278072415841521, "imbalance_abs_log_ratio": 0.8233296537390045, "last_quartile_mean": 0.00013055396993877366, "mean": 0.00019328556300024502, "normalized": [ 1.889398576482247, 1.780233894901915, 1.6772594716902607, 1.566224958959574, 1.4762232610465842, 1.375571144634086, 1.3083686481730985, 1.2364401527267181, 1.173148897934763, 1.1508864184295666, 1.080457040307468, 1.0318582947152708, 1.0081957004611959, 0.9698994715426978, 0.9066127339785177, 0.8754550048258108, 0.8446146862114313, 0.8115724449181009, 0.7806677558079268, 0.7613480996239201, 0.7403134026279838, 0.7294052762839086, 0.7160186976339193, 0.7062575707090604, 0.6942777320753531, 0.6866005528973866, 0.6829938476675228, 0.6753403086482458, 0.6715967314167978, 0.6686860556999406, 0.6640286432895989, 0.6600445236791291 ], "peak_layer": 1, "peak_normalized": 1.889398576482247, "population_cv": 0.3579154260008552, "rest_mean": 0.00020340604748559632, "spike_contrast": 0.681567477924495, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013863494677934796, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0003651934675872326, 0.0003440935106482357, 0.00032419004128314555, 0.0003027286729775369, 0.0002853326441254467, 0.00026587804313749075, 0.00025288877077400684, 0.00023898603103589267, 0.00022675274522043765, 0.00022244972933549434, 0.0002088367473334074, 0.00019944331143051386, 0.00019486967357806861, 0.00018746756541077048, 0.00017523515271022916, 0.00016921281348913908, 0.0001632518251426518, 0.00015686523693148047, 0.00015089180669747293, 0.00014715759607497603, 0.00014309189282357693, 0.00014098350948188454, 0.00013839607709087431, 0.0001365093921776861, 0.0001341938623227179, 0.00013270997442305088, 0.00013201285037212074, 0.00013053353177383542, 0.0001298099523410201, 0.00012924736074637622, 0.00012834715016651899, 0.00012757707736454904 ] } } }, "post_attention_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "31853ffb07b24cc59e774bb20656840b4358b3080b42516d380f2dc0f46404c8", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.0022265110164880753, 0.0040087271481752396, 0.005379301495850086, 0.006410556845366955, 0.0027253651060163975, 0.004838948138058186, 0.006175771821290255, 0.00742313964292407, 0.0030937574338167906, 0.005150026176124811, 0.007019025273621082, 0.008143599145114422, 0.004255065228790045, 0.0066224560141563416, 0.008129785768687725, 0.010099578648805618, 0.0041999719105660915, 0.00719039561226964, 0.009030314162373543, 0.010736005380749702, 0.005131533369421959, 0.00787449348717928, 0.009912566281855106, 0.012113703414797783, 0.0059690922498703, 0.00876442901790142, 0.010806876234710217, 0.012353328987956047, 0.006502739619463682, 0.009470963850617409, 0.012123330496251583, 0.01384514570236206 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.00015962236011546338, "first_to_last_ratio": 3.7698422272252903, "imbalance_abs_log_ratio": 1.3270331510418396, "last_quartile_mean": 4.2341920561739244e-05, "mean": 8.61727822893954e-05, "normalized": [ 2.542734798705971, 2.3221448837042376, 2.101145461274622, 1.8911858524935854, 1.7004204056817789, 1.5354142426015858, 1.4229001770142031, 1.3028733810328659, 1.2049412416934622, 1.1705258152816296, 1.0773908232649334, 1.0133220774376726, 0.990609360100461, 0.9272534949358001, 0.8550385909364457, 0.8185651575045715, 0.7703972051466339, 0.7173947009730917, 0.6760256695667757, 0.6514523109028573, 0.618419077603279, 0.6053620388084041, 0.5838930841859182, 0.5697043695032278, 0.5457802700901755, 0.5276149005446632, 0.5104701332743111, 0.49517988582787786, 0.47527562953524777, 0.46579059316028215, 0.45896435764104826, 0.45181000957238127 ], "peak_layer": 1, "peak_normalized": 2.542734798705971, "population_cv": 0.5705202801130907, "rest_mean": 9.280119640472414e-05, "spike_contrast": 0.542873885449774, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.03793460666202e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00021911453222855926, 0.00020010568550787866, 0.00018106155039276928, 0.00016296874673571438, 0.00014652995741926134, 0.00013231091725174338, 0.0001226152671733871, 0.00011227222421439365, 0.00010383313929196447, 0.00010086746624438092, 9.284176485380158e-05, 8.732078276807442e-05, 8.536356472177431e-05, 7.99040135461837e-05, 7.368105434579775e-05, 7.053803710732609e-05, 6.638727063545957e-05, 6.181989738252014e-05, 5.825501284562051e-05, 5.613745815935545e-05, 5.329089253791608e-05, 5.216573117650114e-05, 5.0315691623836756e-05, 4.9093010602518916e-05, 4.703140439232811e-05, 4.546604395727627e-05, 4.398863165988587e-05, 4.267102849553339e-05, 4.095582335139625e-05, 4.013847137684934e-05, 3.955023566959426e-05, 3.893372559105046e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.00021442477373057045, "first_to_last_ratio": 1.8431254562763404, "imbalance_abs_log_ratio": 0.6114627481432037, "last_quartile_mean": 0.00011633759004325839, "mean": 0.00015221248349917005, "normalized": [ 1.7740805953138137, 1.6484457246979003, 1.5301434809496548, 1.4217241600875643, 1.3264958046529274, 1.2454577490392524, 1.1897861330408546, 1.1336265611690741, 1.086914920172498, 1.064301358569379, 1.0227209980571423, 0.9941753882589759, 0.9788979908211966, 0.9511107967643203, 0.9228143274453928, 0.9052447611165039, 0.8846530997418666, 0.8652930871753776, 0.8497381552743479, 0.8388656494639406, 0.8256334792987466, 0.8176309647342147, 0.8076951735882674, 0.8000661307359944, 0.7892605467734257, 0.78144613040264, 0.7731219607566042, 0.7664988488799842, 0.7584935618388989, 0.7529639051546407, 0.7485195296370439, 0.7441790263875565 ], "peak_layer": 1, "peak_normalized": 1.7740805953138137, "population_cv": 0.2736411578809574, "rest_mean": 0.00015762286892384981, "spike_contrast": 0.7803207938393821, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012299640220589936, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00027003721334040165, 0.00025091401766985655, 0.00023290693934541196, 0.0002164041652576998, 0.000201909220777452, 0.0001895742170745507, 0.00018110030214302242, 0.00017255211423616856, 0.00016544201935175806, 0.00016199995297938585, 0.00015567090304102749, 0.00015132590488065034, 0.0001490004942752421, 0.00014477093645837158, 0.00014046386058907956, 0.00013778955326415598, 0.0001346552453469485, 0.00013170840975362808, 0.00012934075493831187, 0.00012768582382705063, 0.00012567172234412283, 0.00012445363972801715, 0.00012294128828216344, 0.00012178005272289738, 0.00012013530795229599, 0.00011894585622940212, 0.00011767881369451061, 0.00011667069338727742, 0.00011545218876563013, 0.00011461050598882139, 0.00011393401655368507, 0.00011327333777444437 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.190154030919075, "first_to_last_ratio": 1.84312545379361, "imbalance_abs_log_ratio": 0.6114627467961817, "last_quartile_mean": 0.10316933691501617, "mean": 0.13498355145566165, "normalized": [ 1.774080628234443, 1.6484456620184311, 1.5301434020504485, 1.4217242444591272, 1.326495766996536, 1.2454576834375484, 1.189786122599316, 1.133626627255238, 1.0869149569673349, 1.0643014004698326, 1.0227209907477024, 0.9941753870542451, 0.9788979589698585, 0.9511107512164827, 0.9228143032197887, 0.9052447972745712, 0.8846531331847195, 0.8652931173486506, 0.8497382179112226, 0.8388656682549475, 0.8256334805754935, 0.8176309131099365, 0.8076951535479161, 0.8000661540388709, 0.7892605573866218, 0.7814461538045975, 0.7731219580171443, 0.7664988542586116, 0.7584935821787124, 0.7529638601652359, 0.7485195164462104, 0.7441789968002049 ], "peak_layer": 1, "peak_normalized": 1.774080628234443, "population_cv": 0.27364115345546325, "rest_mean": 0.13978153549962574, "spike_contrast": 0.7803207857775153, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.10907443761825561, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.2394717037677765, 0.22251304984092712, 0.2065441906452179, 0.19190938770771027, 0.1790551096200943, 0.16811630129814148, 0.16060155630111694, 0.1530209481716156, 0.14671564102172852, 0.1436631828546524, 0.13805051147937775, 0.13419732451438904, 0.13213512301445007, 0.1283843070268631, 0.1245647519826889, 0.12219315767288208, 0.11941362172365189, 0.11680033802986145, 0.11470068246126175, 0.1132330670952797, 0.11144693940877914, 0.11036672443151474, 0.10902556031942368, 0.10799577087163925, 0.10653719305992126, 0.10548237711191177, 0.1043587476015091, 0.10346473753452301, 0.10238415747880936, 0.10163773596286774, 0.10103782266378403, 0.10045192390680313 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 6.883367768750759e-05, "first_to_last_ratio": 2.249759357551583, "imbalance_abs_log_ratio": 0.810823258297238, "last_quartile_mean": 3.059601795030176e-05, "mean": 4.691846504556452e-05, "normalized": [ 1.7359059710084281, 1.6475789727938472, 1.5536456441204052, 1.4921194549590373, 1.4365657321885223, 1.3438513058553838, 1.2787343947807137, 1.2483299149152907, 1.1912354282513808, 1.1603083401306369, 1.099547144002948, 1.059972440022979, 1.0583955432739494, 1.0176800955770247, 0.9754963047746087, 0.9631595708780456, 0.9242400656751482, 0.8752301116137658, 0.8454245377074081, 0.8232853322610043, 0.7863494898171388, 0.7774642190168813, 0.7497486129019063, 0.7388483538626245, 0.709315327139865, 0.6908343794807076, 0.6685469983108738, 0.6592365073812129, 0.6314341746588822, 0.6235898169926835, 0.6199474545603838, 0.6139783610863121 ], "peak_layer": 1, "peak_normalized": 1.7359059710084281, "population_cv": 0.3220406003070692, "rest_mean": 4.9070239684725595e-05, "spike_contrast": 0.7193541792517962, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.52988819940947e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 8.144604362314567e-05, 7.730187644483522e-05, 7.28946688468568e-05, 7.000795449130237e-05, 6.740145909134299e-05, 6.305144052021205e-05, 5.999625500408001e-05, 5.8569723478285596e-05, 5.589093780145049e-05, 5.4439886298496276e-05, 5.158906424185261e-05, 4.973227987647988e-05, 4.9658294301480055e-05, 4.774798799189739e-05, 4.576878927764483e-05, 4.51899686595425e-05, 4.3363925215089694e-05, 4.1064453398576006e-05, 3.9666021621087566e-05, 3.86272840842139e-05, 3.689431105158292e-05, 3.6477427784120664e-05, 3.5177054087398574e-05, 3.466563066467643e-05, 3.3279986382694915e-05, 3.2412888685939834e-05, 3.136719897156581e-05, 3.093036502832547e-05, 2.9625922252307646e-05, 2.9257877031341195e-05, 2.9086982976878062e-05, 2.8806922273361124e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 0.00017341253987979144, "first_to_last_ratio": 1.4955148931299893, "imbalance_abs_log_ratio": 0.4024705576674927, "last_quartile_mean": 0.00011595507385209203, "mean": 0.00013990553861731314, "normalized": [ 1.3768818258178377, 1.3340879911950096, 1.2910054180628048, 1.2527236427111486, 1.2147542166907819, 1.1769889670731195, 1.1474411155803916, 1.1220953663134672, 1.0944075691302455, 1.0773007485615413, 1.0508678613623372, 1.0320596054419082, 1.0208161736522385, 1.000559648637946, 0.9805197815181176, 0.9663128275349431, 0.948744191771126, 0.9319305822463463, 0.9182019814348666, 0.9075038868226692, 0.8943650364968407, 0.886262676120959, 0.8757907045344768, 0.8679002174812178, 0.856288373659771, 0.8478178092769747, 0.8387215581135022, 0.8313975747674917, 0.8224084044020752, 0.8161864847722404, 0.8112200992010112, 0.8064376596145924 ], "peak_layer": 1, "peak_normalized": 1.3768818258178377, "population_cv": 0.16325888721442564, "rest_mean": 0.0001431150389921472, "spike_contrast": 0.8564734877369171, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001225742365932092, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0001926333934534341, 0.00018664629897102714, 0.00018061880837194622, 0.0001752629759721458, 0.00016995084297377616, 0.0001646672753849998, 0.00016053336730692536, 0.00015698735660407692, 0.00015311368042603135, 0.00015072034148033708, 0.00014702223415952176, 0.00014439085498452187, 0.0001428178366040811, 0.00013998383656144142, 0.00013718014815822244, 0.00013519251660909504, 0.00013273456715978682, 0.00013038225006312132, 0.00012846154277212918, 0.00012696482008323073, 0.00012512662215158343, 0.0001239930570591241, 0.00012252797023393214, 0.000121424047392793, 0.00011979948612861335, 0.00011861440725624561, 0.00011734179133782163, 0.0001163171255029738, 0.00011505949078127742, 0.00011418900976423174, 0.00011349418491590768, 0.00011282509512966499 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 0.00014675183047074825, "first_to_last_ratio": 1.2676545313577883, "imbalance_abs_log_ratio": 0.23716836728277432, "last_quartile_mean": 0.00011576642282307148, "mean": 0.00013041823922321782, "normalized": [ 1.1700698937843759, 1.1560440994564782, 1.1417398043557874, 1.130463200949706, 1.118403537277814, 1.1065304334184665, 1.0939881912841023, 1.0846815122144253, 1.0709468270677047, 1.0579907390333845, 1.0442838370166418, 1.035550673160169, 1.025959021745254, 1.0140203095307772, 1.0062047700154308, 0.9976631870190885, 0.9861765922450596, 0.9770016877798172, 0.9687702939454803, 0.9602422119882037, 0.9504995944091904, 0.9425883202531964, 0.9330014667281973, 0.9259385264566372, 0.9147443798098084, 0.906202796813466, 0.8958065504175365, 0.8885655303249116, 0.8806194435723184, 0.8757905905296257, 0.8713783728577705, 0.8681336045391749 ], "peak_layer": 1, "peak_normalized": 1.1700698937843759, "population_cv": 0.09121429133712369, "rest_mean": 0.0001320278307604086, "spike_contrast": 0.921975648780332, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001217264449223876, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0001525984553154558, 0.0001507692359155044, 0.000148903694935143, 0.00014743302017450333, 0.00014586022007279098, 0.00014431175077334046, 0.00014267601363826543, 0.00014146225294098258, 0.000139670999487862, 0.00013798128929920495, 0.00013619365927297622, 0.00013505469541996717, 0.00013380376913119107, 0.00013224674330558628, 0.00013122745440341532, 0.00013011347618885338, 0.00012861541472375393, 0.00012741883983835578, 0.0001263453159481287, 0.0001252330985153094, 0.0001239624834852293, 0.00012293070903979242, 0.00012168040848337114, 0.00012075927224941552, 0.00011929935135412961, 0.00011818537313956767, 0.00011682951299007982, 0.00011588515189941972, 0.00011484883725643158, 0.00011421906674513593, 0.00011364363308530301, 0.00011322045611450449 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00030004180916876066, "first_to_last_ratio": 2.2837506327398005, "imbalance_abs_log_ratio": 0.8258191057718037, "last_quartile_mean": 0.00013138116082700435, "mean": 0.00019473525253488333, "normalized": [ 1.9000096511818072, 1.783700741635724, 1.6708066910504589, 1.570366884717585, 1.4783058799917985, 1.376185321596543, 1.3064973297630273, 1.2402700043336092, 1.1767953849206394, 1.1505341943958838, 1.0771542571420398, 1.026098162637436, 1.013898063759585, 0.9669473025647412, 0.9095398902231254, 0.8755205777387188, 0.842301214161826, 0.8073124727024382, 0.7791991818813049, 0.760574607765185, 0.7402055397447304, 0.7308143413325056, 0.7152667122155245, 0.7043714155678553, 0.6942467749765324, 0.6868749151283189, 0.680655564574882, 0.676363713518914, 0.6701061029149729, 0.6672276319278244, 0.6636177363814285, 0.6582317375530352 ], "peak_layer": 1, "peak_normalized": 1.9000096511818072, "population_cv": 0.35930746026536914, "rest_mean": 0.00020494150898971216, "spike_contrast": 0.6812747128050888, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013962146767880766, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0003699988592416048, 0.0003473494143690914, 0.0003253649629186839, 0.0003058057918678969, 0.0002878782688640058, 0.0002679917961359024, 0.0002544210874475539, 0.00024152429250534624, 0.00022916354646440595, 0.000224049566895701, 0.00020975990628357977, 0.00019981748482678086, 0.000197441695490852, 0.0001882987271528691, 0.00017711948021315038, 0.00017049472080543637, 0.00016402573965024203, 0.00015721219824627042, 0.00015173754945863038, 0.00014811068831477314, 0.00014414411270990968, 0.0001423153153154999, 0.0001392876438330859, 0.00013716594548895955, 0.00013519432104658335, 0.00013375876005738974, 0.00013254763325676322, 0.0001317118585575372, 0.00013049328117631376, 0.00012993274140171707, 0.00012922976748086512, 0.00012818092363886535 ] } } }, "post_mlp_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "7fd9c911e93175896ad618842c120e77b3f017c17b7eec9c2bfb01fc6dbc8ac2", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.0034885562490671873, 0.004831995815038681, 0.005952076520770788, 0.006925336085259914, 0.00385080324485898, 0.005542940925806761, 0.006729288958013058, 0.007927131839096546, 0.004081308841705322, 0.005811207927763462, 0.007534808944910765, 0.008581922389566898, 0.00499920267611742, 0.007102519739419222, 0.008542952127754688, 0.010447698645293713, 0.005041283555328846, 0.007636935915797949, 0.009387043304741383, 0.011020516976714134, 0.005807018838822842, 0.008367395959794521, 0.01024632342159748, 0.012378672137856483, 0.006453148555010557, 0.009127611294388771, 0.011037957854568958, 0.012611051090061665, 0.007010402157902718, 0.009794782847166061, 0.012445803731679916, 0.014087481424212456 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.00015858902861509705, "first_to_last_ratio": 3.757869257587208, "imbalance_abs_log_ratio": 1.3238521099622325, "last_quartile_mean": 4.220184837322449e-05, "mean": 8.57325238712292e-05, "normalized": [ 2.5462257183789148, 2.3032637538343064, 2.0924289533379263, 1.8886492391629932, 1.7031256747388024, 1.5374176267930122, 1.4224231993460965, 1.3049597038569594, 1.2055618292025094, 1.169245405308266, 1.0769812048488787, 1.0144334047898542, 0.9913512297333369, 0.9317090592952362, 0.8577499774884012, 0.81555804918997, 0.7682725064221458, 0.7217807351250844, 0.6783324192661732, 0.6511800118936032, 0.6192916242842709, 0.6053728292385717, 0.5848720876072855, 0.5718124973482859, 0.5462302773879167, 0.5275173250132701, 0.5112998770983732, 0.49691833866078555, 0.47721658788927634, 0.46551989467389665, 0.46012635628943105, 0.45317260249616614 ], "peak_layer": 1, "peak_normalized": 2.5462257183789148, "population_cv": 0.5686111305901927, "rest_mean": 9.231303704803361e-05, "spike_contrast": 0.5437775023084314, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.019775271648541e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00021829435718245804, 0.00019746461475733668, 0.0001793892151908949, 0.0001619186659809202, 0.00014601326256524771, 0.00013180669338908046, 0.00012194793089292943, 0.00011187748896190897, 0.00010335585830034688, 0.00010024235962191597, 9.233231685357168e-05, 8.69699360919185e-05, 8.499104296788573e-05, 7.987776916706935e-05, 7.353707042057067e-05, 6.991984992055222e-05, 6.586594099644572e-05, 6.188008410390466e-05, 5.8155150327365845e-05, 5.582730591413565e-05, 5.309343396220356e-05, 5.190014053368941e-05, 5.014256021240726e-05, 4.90229285787791e-05, 4.682970029534772e-05, 4.522539165918715e-05, 4.383502891869284e-05, 4.2602063331287354e-05, 4.0912982512963936e-05, 3.991019548266195e-05, 3.944779382436536e-05, 3.885163096128963e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.00021242368711682502, "first_to_last_ratio": 1.8345658968465297, "imbalance_abs_log_ratio": 0.6068078850437921, "last_quartile_mean": 0.0001157896194854402, "mean": 0.0001511171253696375, "normalized": [ 1.7717451736709768, 1.636674793183293, 1.5242755762025908, 1.4186515254985965, 1.3264266232459345, 1.2453738409548603, 1.1893512719369177, 1.1330135273991528, 1.085860457802012, 1.0628542818802846, 1.023252039669589, 0.9942070699734702, 0.9784308644775869, 0.9531083940389613, 0.9232390376562555, 0.9040072643238835, 0.8853775311228679, 0.8680181295617843, 0.8515367513400655, 0.8395795332993111, 0.8267439067367663, 0.8185133286645813, 0.8091730400239778, 0.800791325997512, 0.7909899739933552, 0.7821072339266707, 0.7753403490721149, 0.7682422073314642, 0.7604971517068638, 0.7549863467424895, 0.7508596466770359, 0.7467718018887739 ], "peak_layer": 1, "peak_normalized": 1.7717451736709768, "population_cv": 0.2714186609524326, "rest_mean": 0.00015645541295978345, "spike_contrast": 0.7816308178118705, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012229037238284946, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0002677410375326872, 0.0002473295899108052, 0.00023034414334688336, 0.00021438254043459892, 0.00020044577831868082, 0.00018819731485564262, 0.00017973134526982903, 0.000171217747265473, 0.0001640921109355986, 0.00016061548376455903, 0.00015463090676348656, 0.00015024171443656087, 0.0001478576596127823, 0.00014403100067283958, 0.00013951722939964384, 0.00013661097909789532, 0.00013379570737015456, 0.0001311724045081064, 0.0001286817860091105, 0.00012687484559137374, 0.0001249351626029238, 0.00012369138130452484, 0.00012227990373503417, 0.00012101328320568427, 0.00011953213106608018, 0.0001181897969217971, 0.0001171672047348693, 0.00011609455395955592, 0.00011492414341773838, 0.0001140913664130494, 0.00011346775136189535, 0.00011285000800853595 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.18837944976985455, "first_to_last_ratio": 1.8345658800697515, "imbalance_abs_log_ratio": 0.6068078758989703, "last_quartile_mean": 0.10268339328467846, "mean": 0.1340121782850474, "normalized": [ 1.7717451751532867, 1.6366747548098712, 1.5242756140248972, 1.4186514377480737, 1.3264266357207033, 1.2453739091726768, 1.1893513017871604, 1.1330134634090196, 1.0858604665419038, 1.0628542749024097, 1.0232520332438115, 0.9942070832344971, 0.9784308561813967, 0.9531084116998004, 0.9232390798282929, 0.9040072094634526, 0.8853775581660035, 0.8680181154203164, 0.8515367049614931, 0.8395794986042229, 0.8267439263823189, 0.8185133396785964, 0.809173050926764, 0.8007913534932437, 0.7909900046670136, 0.782107217826699, 0.7753403703776178, 0.7682422246066585, 0.7604971603471317, 0.7549863443945886, 0.7508596538022533, 0.7467717694238248 ], "peak_layer": 1, "peak_normalized": 1.7717451751532867, "population_cv": 0.27141865758333467, "rest_mean": 0.13874622461972413, "spike_contrast": 0.7816308398662988, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.10844832807779312, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.23743543028831482, 0.21933434903621674, 0.20427149534225464, 0.19011656939983368, 0.17775732278823853, 0.16689527034759521, 0.15938755869865417, 0.15183760225772858, 0.14551852643489838, 0.14243541657924652, 0.13712823390960693, 0.1332358568906784, 0.1311216503381729, 0.12772813439369202, 0.1237252801656723, 0.12114797532558441, 0.1186513751745224, 0.1163249984383583, 0.11411628872156143, 0.11251387745141983, 0.11079375445842743, 0.10969075560569763, 0.10843904316425323, 0.107315793633461, 0.10600229352712631, 0.104811891913414, 0.10390505194664001, 0.10295381397008896, 0.1019158810377121, 0.10117736458778381, 0.10062433779239655, 0.10007651150226593 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 6.873207303215167e-05, "first_to_last_ratio": 2.248757814140914, "imbalance_abs_log_ratio": 0.8103779811585758, "last_quartile_mean": 3.0564462122129044e-05, "mean": 4.6857780716891284e-05, "normalized": [ 1.7400119939228862, 1.6405409364325143, 1.555121247708426, 1.4944054279879166, 1.4327475398601994, 1.3414100714975594, 1.2804101716466747, 1.2499370459040815, 1.192804641985012, 1.1608858830017554, 1.098191983597808, 1.057744066133373, 1.0542717513613309, 1.021869458338179, 0.9775749297615711, 0.9570927445976433, 0.9210091333285576, 0.8815125232807631, 0.847820728512489, 0.8204970180441978, 0.7871838673851995, 0.7759041238100245, 0.7506938949785397, 0.7421070509174572, 0.7072987335050381, 0.6891811912847667, 0.6695407652827761, 0.6616333377893114, 0.6345027924930273, 0.6212629426015038, 0.6200302724102036, 0.614801730639213 ], "peak_layer": 1, "peak_normalized": 1.7400119939228862, "population_cv": 0.32175451551289735, "rest_mean": 4.9004235565748616e-05, "spike_contrast": 0.7196709452950106, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.5266924533061685e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 8.153310045599937e-05, 7.687210745643824e-05, 7.28695304132998e-05, 7.002452184678987e-05, 6.713537004543468e-05, 6.28554989816621e-05, 5.9997179050697014e-05, 5.8569276006892323e-05, 5.589217835222371e-05, 5.439653614303097e-05, 5.1458839152473956e-05, 4.9563539505470544e-05, 4.9400834541302174e-05, 4.788253500009887e-05, 4.58069916930981e-05, 4.4847241952084005e-05, 4.315644400776364e-05, 4.130572051508352e-05, 3.972699778387323e-05, 3.8446669350378215e-05, 3.688568904181011e-05, 3.635714529082179e-05, 3.5175849916413426e-05, 3.477348946034908e-05, 3.3142448955914006e-05, 3.2293501135427505e-05, 3.13731943606399e-05, 3.1002669857116416e-05, 2.9731392714893445e-05, 2.9111002731951885e-05, 2.905324254243169e-05, 2.8808244678657502e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 0.00017190765356644988, "first_to_last_ratio": 1.4894621368061003, "imbalance_abs_log_ratio": 0.3984150727783497, "last_quartile_mean": 0.00011541592721187044, "mean": 0.00013894661265112518, "normalized": [ 1.3739040544627474, 1.3296102750467884, 1.2883342981472405, 1.2499918159027226, 1.2130627732820631, 1.175544413473833, 1.1474083142393565, 1.1199112790671786, 1.092909618037032, 1.0755929927697745, 1.0512861461262244, 1.0312571117882559, 1.0194388253067554, 1.0019061415077835, 0.980344379776871, 0.9647961249933756, 0.9493075664595542, 0.9344817410273759, 0.9198929201794295, 0.9079992279088673, 0.8952920956888009, 0.8869530532606651, 0.8771902552778059, 0.8683889858357474, 0.8579407806493952, 0.8482823463732029, 0.8408658204630546, 0.8330843095877946, 0.8243382853592501, 0.818115422616924, 0.8135611224181157, 0.8090075029660142 ], "peak_layer": 1, "peak_normalized": 1.3739040544627474, "population_cv": 0.16161403917150835, "rest_mean": 0.00014210757001786045, "spike_contrast": 0.8576421569620596, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012187744287075474, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00019089931447524577, 0.00018474484386388212, 0.00017900968668982387, 0.00017368212866131216, 0.0001685509632807225, 0.00016333791427314281, 0.00015942849859129637, 0.00015560787869617343, 0.00015185608936008066, 0.00014945000293664634, 0.00014607264893129468, 0.0001432896824553609, 0.00014164757158141583, 0.0001392114645568654, 0.00013621553080156446, 0.0001340551534667611, 0.00013190307072363794, 0.00012984307250007987, 0.0001278160052606836, 0.00012616341700777411, 0.0001243978040292859, 0.00012323912233114243, 0.00012188261462142691, 0.00012065970804542303, 0.00011920796532649547, 0.00011786595860030502, 0.00011683545744745061, 0.00011575424287002534, 0.00011453901242930442, 0.00011367436673026532, 0.00011304156214464456, 0.00011240885214647278 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 0.00014552358334185556, "first_to_last_ratio": 1.262638843036328, "imbalance_abs_log_ratio": 0.2332038508028458, "last_quartile_mean": 0.00011525352965691127, "mean": 0.00012953498708156985, "normalized": [ 1.1676304712442283, 1.1525836984023852, 1.1391521450612143, 1.1288261090295841, 1.1167600441451297, 1.1054423860149858, 1.094731137320412, 1.082319740349084, 1.067957453000188, 1.0563935463552665, 1.04560972624621, 1.0337959762176818, 1.0235535208863231, 1.0159040893770892, 1.0050628637064216, 0.9950792389307415, 0.9865814183778941, 0.9786230528286792, 0.9704418054120744, 0.9608381133357455, 0.9508407831226554, 0.9429749854457838, 0.9346839821902475, 0.9262276149685831, 0.9170542960948607, 0.9059488475652709, 0.8984648923618161, 0.8908037775288885, 0.8831483920181569, 0.8778579808382188, 0.8744051093280881, 0.8703028022960919 ], "peak_layer": 1, "peak_normalized": 1.1676304712442283, "population_cv": 0.0896642712448568, "rest_mean": 0.00013110964479252557, "spike_contrast": 0.9231344927669946, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012103183544240892, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00015124899800866842, 0.00014929991448298097, 0.00014756005839444697, 0.00014622247545048594, 0.00014465949789155275, 0.00014319346519187093, 0.00014180598373059183, 0.00014019827358424664, 0.00013833785487804562, 0.00013683992438018322, 0.0001354430423816666, 0.00013391274842433631, 0.0001325859921053052, 0.00013159512309357524, 0.00013019080506637692, 0.00012889757636003196, 0.00012779681128449738, 0.00012676592450588942, 0.00012570616672746837, 0.00012446215259842575, 0.00012316714855842292, 0.00012214825255796313, 0.00012107427755836397, 0.00011997888213954866, 0.00011879061639774591, 0.00011735207226593047, 0.0001163826382253319, 0.0001153902558144182, 0.00011439861555118114, 0.00011371332220733166, 0.00011326605454087257, 0.0001127346622524783 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00029742828155576717, "first_to_last_ratio": 2.2781890121807993, "imbalance_abs_log_ratio": 0.8233808344454435, "last_quartile_mean": 0.00013055469935352448, "mean": 0.00019329176166138495, "normalized": [ 1.8891850069388627, 1.7801603926271052, 1.6773544463608039, 1.566500262900107, 1.476456430986117, 1.3754925511314788, 1.3084504582544147, 1.236424592515421, 1.172965600373877, 1.150871569148191, 1.0802294364746856, 1.0319209662753672, 1.0080063247818516, 0.9698985570693867, 0.9067931019504507, 0.8756163462587222, 0.8444435806568995, 0.8113939671049883, 0.7809677247391495, 0.7613442367276676, 0.7402582678385915, 0.7293562882295931, 0.7162426695419845, 0.7062419232587185, 0.6940340549797547, 0.6866100033591334, 0.6831688143117768, 0.6753320518823956, 0.6715919825156708, 0.6686410475337363, 0.6640239865195754, 0.6600233567535221 ], "peak_layer": 1, "peak_normalized": 1.8891850069388627, "population_cv": 0.357915029863326, "rest_mean": 0.00020341356920888993, "spike_contrast": 0.6815376252628051, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001386340009048581, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00036516389809548855, 0.0003440903383307159, 0.0003242187958676368, 0.0003027915954589844, 0.00028538686456158757, 0.00026587137836031616, 0.0002529126941226423, 0.00023899068764876574, 0.00022672458726447076, 0.00022245399304665625, 0.0002087994507746771, 0.00019946182146668434, 0.00019483931828290224, 0.00018747340072877705, 0.00017527563613839447, 0.00016924942610785365, 0.00016322398732881993, 0.00015683576930314302, 0.0001509546273155138, 0.00014716156874783337, 0.0001430858246749267, 0.0001409785618307069, 0.00013844380737282336, 0.00013651074550580233, 0.00013415106514003128, 0.00013271605712361634, 0.00013205090363044292, 0.00013053612201474607, 0.00012981319741811603, 0.00012924280599690974, 0.00012835036613978446, 0.00012757707736454904 ] } } }, "pre_attention_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "de71d4cf2d509a1063f6eac7f50c4885e9a058dee7e4b993ad40f5527f528d00", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.02802751213312149, 0.01413465291261673, 0.014225807972252369, 0.01433455292135477, 0.01440478302538395, 0.009703479707241058, 0.00977881159633398, 0.009858808480203152, 0.009950309991836548, 0.00752287870272994, 0.007571155205368996, 0.007693085819482803, 0.007788818795233965, 0.006353048607707024, 0.006389216054230928, 0.0064707002602517605, 0.00656962301582098, 0.005553483963012695, 0.0056249531917274, 0.005688921548426151, 0.005768724251538515, 0.004966245498508215, 0.005057469941675663, 0.005098348017781973, 0.005216081161051989, 0.0046758754178881645, 0.004727107938379049, 0.004777040798217058, 0.004837613552808762, 0.0044186655431985855, 0.004510571248829365, 0.0045849340967834 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.0001580166108396952, "first_to_last_ratio": 1.9713740353643698, "imbalance_abs_log_ratio": 0.6787307795042399, "last_quartile_mean": 8.015557068574708e-05, "mean": 0.00011843875881822896, "normalized": [ 0.8918679597819287, 1.5814819593588525, 1.5722239987443252, 1.3246067564285042, 1.1252650527356378, 1.512743589128775, 1.3788112428745087, 1.2863035124436846, 1.097095055843533, 1.468154353173571, 1.1523254606068964, 1.0839888571909224, 1.054518217028451, 1.146335080549274, 1.1665208527784754, 0.8931136212023528, 0.8203804970547371, 1.073741223277434, 0.919654742394771, 0.8223971330190447, 0.7711288002244273, 0.8470775332584999, 0.8298711629738247, 0.7662487463744663, 0.7210382462310316, 0.7071376059209309, 0.7189559392079661, 0.727980518763353, 0.6111916361855597, 0.6986829946379066, 0.6244678178312321, 0.6046898327751223 ], "peak_layer": 2, "peak_normalized": 1.5814819593588525, "population_cv": 0.2904715201464359, "rest_mean": 0.00012310891098523926, "spike_contrast": 0.7572151875143335, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.321993711637333e-05, "top_five_layers": [ 2, 3, 6, 10, 7 ], "top_five_spike_overlap": 0 }, "values": [ 0.00010563173418631777, 0.0001873087603598833, 0.00018621225899551064, 0.00015688478015363216, 0.0001332749961875379, 0.00017916747310664505, 0.00016330469225067645, 0.00015234819147735834, 0.00012993857671972364, 0.00017388637934345752, 0.0001364799973089248, 0.0001283862948184833, 0.00012489582877606153, 0.00013577050413005054, 0.00013816128193866462, 0.00010577926877886057, 9.71648478298448e-05, 0.00012717257777694613, 0.00010892276623053476, 9.74036956904456e-05, 9.133153798757121e-05, 0.0001003268116619438, 9.828891052166e-05, 9.07535504666157e-05, 8.539887494407594e-05, 8.375250035896897e-05, 8.515224908478558e-05, 8.622110908618197e-05, 7.238877878990024e-05, 8.275114669231698e-05, 7.396119326585904e-05, 7.161871326388791e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.0001664285200604354, "first_to_last_ratio": 1.9719337364651075, "imbalance_abs_log_ratio": 0.6790146534176558, "last_quartile_mean": 8.439863722742302e-05, "mean": 0.0001245867656507471, "normalized": [ 0.8984334461398115, 1.5927952483854981, 1.5658595485525297, 1.3198735865176374, 1.131213894321221, 1.5125315092450449, 1.3796810696180641, 1.286366052942558, 1.0960606282761096, 1.4536884557404346, 1.1543342746982923, 1.0797559630746076, 1.0497530570909623, 1.1457182987306291, 1.1551962694199982, 0.9023721663305979, 0.8238710779144349, 1.0632503993742801, 0.9211624832492573, 0.8274335806257342, 0.7739032412019898, 0.850990856837025, 0.8281317613098801, 0.7681943949939335, 0.7208251570035091, 0.7089546837555952, 0.7203162530736348, 0.7236174125342815, 0.6145740290254166, 0.6994991973458914, 0.6274381906330746, 0.6042038120380651 ], "peak_layer": 2, "peak_normalized": 1.5927952483854981, "population_cv": 0.2889665774838781, "rest_mean": 0.00012946851159169995, "spike_contrast": 0.7586812913967177, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.822533756960183e-05, "top_five_layers": [ 2, 3, 6, 10, 7 ], "top_five_spike_overlap": 0 }, "values": [ 0.00011193291720701382, 0.0001984412083402276, 0.0001950853766174987, 0.000164438781212084, 0.00014093428035266697, 0.00018844140868168324, 0.00017189000209327787, 0.00016026418597903103, 0.00013655464863404632, 0.00018111034296453, 0.00014381477376446128, 0.0001345233031315729, 0.00013078533811494708, 0.00014274133718572557, 0.00014392216689884663, 0.00011242362961638719, 0.00010264343291055411, 0.0001324669283349067, 0.00011476465442683548, 0.00010308727360097691, 9.641810174798593e-05, 0.00010602219845168293, 0.00010317425767425448, 9.570685506332666e-05, 8.980527491075918e-05, 8.832637104205787e-05, 8.974187221610919e-05, 9.015315299620852e-05, 7.656779052922502e-05, 8.714834257261828e-05, 7.817049481673166e-05, 7.527579873567447e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.14759047608822584, "first_to_last_ratio": 1.9719337947467823, "imbalance_abs_log_ratio": 0.6790146829732503, "last_quartile_mean": 0.07484555337578058, "mean": 0.1104847895912826, "normalized": [ 0.8984334974418968, 1.5927952138130326, 1.565859646322909, 1.3198735575362357, 1.1312139236939245, 1.5125315007911495, 1.3796810264894532, 1.286366010451999, 1.0960606168617568, 1.4536884922319049, 1.1543342013148132, 1.0797558953237962, 1.0497530983161607, 1.1457182562234747, 1.1551962948455465, 0.9023721940039697, 0.8238711061937815, 1.0632504861804335, 0.9211625151546727, 0.8274335809310764, 0.7739032642161865, 0.8509909070635955, 0.8281317391228538, 0.7681943896830757, 0.7208251689669645, 0.7089546589522648, 0.7203162343708721, 0.7236173971219663, 0.6145740294939214, 0.6994991437375198, 0.6274381319743458, 0.6042038211744466 ], "peak_layer": 2, "peak_normalized": 1.5927952138130326, "population_cv": 0.28896658018674404, "rest_mean": 0.11481397047086998, "spike_contrast": 0.7586813040631772, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.08710721284151077, "top_five_layers": [ 2, 3, 6, 10, 7 ], "top_five_spike_overlap": 0 }, "values": [ 0.09926323592662811, 0.1759796440601349, 0.1730036735534668, 0.14582595229148865, 0.12498193234205246, 0.16711172461509705, 0.15243376791477203, 0.14212387800216675, 0.12109802663326263, 0.16061046719551086, 0.1275363713502884, 0.1192966029047966, 0.11598175019025803, 0.12658444046974182, 0.12763161957263947, 0.09969840198755264, 0.0910252258181572, 0.11747300624847412, 0.10177444666624069, 0.09141882508993149, 0.08550453931093216, 0.09402155131101608, 0.09149596095085144, 0.08487379550933838, 0.0796402171254158, 0.0783287063241005, 0.07958398759365082, 0.07994871586561203, 0.06790108233690262, 0.07728401571512222, 0.069322369992733, 0.06675533205270767 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 5.093040158499207e-05, "first_to_last_ratio": 0.8860093039525244, "imbalance_abs_log_ratio": 0.12102782735710801, "last_quartile_mean": 5.748291960117058e-05, "mean": 5.7093861016710434e-05, "normalized": [ 0.4337136137178416, 0.9185569667519601, 0.9444095926066538, 0.8728787927927439, 0.7849140301431445, 1.1296292591692207, 1.0726593944235638, 0.9796135302182263, 0.9386642177229508, 1.3046353648934683, 1.0461832118506482, 1.0032917361864164, 0.9662905330123907, 1.0760080966276753, 1.1828739682986207, 0.9087955578115051, 0.9194941496879511, 1.198292373503509, 1.0946132296674007, 1.0673695485587213, 0.9244260207272295, 1.0525679458300632, 1.125653431968099, 0.9999504891362015, 0.9629027708994021, 1.0063234987710272, 0.9975933861195996, 1.0791525148711678, 0.9412468232151083, 1.085173475571988, 1.0031556955576197, 0.9789667796878816 ], "peak_layer": 10, "peak_normalized": 1.3046353648934683, "population_cv": 0.1441687540611157, "rest_mean": 5.695535414647397e-05, "spike_contrast": 1.0155638391297448, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.784179811598733e-05, "top_five_layers": [ 10, 18, 15, 6, 23 ], "top_five_spike_overlap": 1 }, "values": [ 2.4762384782661684e-05, 5.2443963795667514e-05, 5.3919990023132414e-05, 4.9836020480142906e-05, 4.481377254705876e-05, 6.449489592341706e-05, 6.124226638348773e-05, 5.592991874436848e-05, 5.3591964388033375e-05, 7.448667020071298e-05, 5.973063889541663e-05, 5.728179894504137e-05, 5.516925739357248e-05, 6.143345672171563e-05, 6.75348419463262e-05, 5.1886647270293906e-05, 5.249747118796222e-05, 6.84151382301934e-05, 6.249569560168311e-05, 6.094024865888059e-05, 5.277905074763112e-05, 6.009516800986603e-05, 6.426790059776977e-05, 5.7091034250333905e-05, 5.4975836974335834e-05, 5.74548939766828e-05, 5.695645813830197e-05, 6.161298369988799e-05, 5.373941530706361e-05, 6.19567435933277e-05, 5.727403186028823e-05, 5.58929932594765e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 5.801741053801379e-05, "first_to_last_ratio": 0.954195857595535, "imbalance_abs_log_ratio": 0.046886327142486896, "last_quartile_mean": 6.08024129178375e-05, "mean": 6.296396202287724e-05, "normalized": [ 0.4796364558374551, 0.9568353419130547, 0.9804638447804884, 0.8949545354119591, 0.8237377234766136, 1.1597846916175047, 1.085649359633547, 0.9904450076993521, 0.9595078978373608, 1.2941748206155441, 1.0680753760655664, 1.0231559828609242, 0.9904019047543832, 1.096490963646882, 1.1850351594596897, 0.9422215946487361, 0.9418631353047864, 1.1705024179981143, 1.0926353878857977, 1.0382275689133567, 0.9320833313178482, 1.0784583323883348, 1.0985080196909318, 0.9917906747078287, 0.9476388143731139, 0.9855066582268529, 0.978396290111904, 1.028407839812513, 0.8982055143685478, 1.0282804954549911, 0.9451124964564701, 0.9138123627295472 ], "peak_layer": 10, "peak_normalized": 1.2941748206155441, "population_cv": 0.13431233496292436, "rest_mean": 6.285090866084729e-05, "spike_contrast": 1.0115120295379672, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.357445017783903e-05, "top_five_layers": [ 10, 15, 18, 6, 23 ], "top_five_spike_overlap": 1 }, "values": [ 3.019981159013696e-05, 6.0246144130360335e-05, 6.173388828756288e-05, 5.634988337988034e-05, 5.186579073779285e-05, 7.302463927771896e-05, 6.835678505012766e-05, 6.236234185053036e-05, 6.041441884008236e-05, 8.148637425620109e-05, 6.725025741616264e-05, 6.442195444833487e-05, 6.235962791834027e-05, 6.903941539349034e-05, 7.461450877599418e-05, 5.9326004702597857e-05, 5.930343468207866e-05, 7.369946979451925e-05, 6.879665306769311e-05, 6.537092122016475e-05, 5.8687659475253895e-05, 6.790400948375463e-05, 6.916641723364592e-05, 6.244707037694752e-05, 5.966709431959316e-05, 6.205140380188823e-05, 6.160370685392991e-05, 6.475263216998428e-05, 5.655457789544016e-05, 6.474461406469345e-05, 5.950802733423188e-05, 5.753724690293893e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 1.7888660295284353e-05, "first_to_last_ratio": 0.41855462748958633, "imbalance_abs_log_ratio": 0.8709478659353102, "last_quartile_mean": 4.2739129185065394e-05, "mean": 3.1986895351110434e-05, "normalized": [ 0.2593955501951102, 0.4997829486650695, 0.48480994473028793, 0.4968760363401999, 0.5474125689219601, 0.7788406177150305, 0.7554473604850604, 0.6514322393717875, 0.7578135841604356, 0.9674810188795085, 0.9032772556875033, 0.9019220651946442, 0.8311356963419073, 0.9587970177975769, 1.1401394869948454, 0.8953886492354074, 1.0262272842074007, 1.1413916918814533, 1.174037958462901, 1.194927715786966, 1.0267140632187253, 1.3434898267078448, 1.3729067467361273, 1.2011929483563188, 1.1735959904166378, 1.3266745689353452, 1.2739952988364465, 1.4049535244931144, 1.2752889026856997, 1.4680979746684197, 1.4182292846928506, 1.3483241791974143 ], "peak_layer": 30, "peak_normalized": 1.4680979746684197, "population_cv": 0.3184955307507873, "rest_mean": 3.066251991657299e-05, "spike_contrast": 1.276428773763581, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.913852269761264e-05, "top_five_layers": [ 30, 31, 28, 23, 32 ], "top_five_spike_overlap": 1 }, "values": [ 8.297258318634704e-06, 1.5986504877218977e-05, 1.5507564967265353e-05, 1.589352177688852e-05, 1.7510028555989265e-05, 2.491269333404489e-05, 2.416441566310823e-05, 2.083729486912489e-05, 2.4240103812189773e-05, 3.094671410508454e-05, 2.889303505071439e-05, 2.8849686714238487e-05, 2.6585450541460887e-05, 3.066893987124786e-05, 3.6469522456172854e-05, 2.8640703021665104e-05, 3.282582474639639e-05, 3.6509576602838933e-05, 3.755382931558415e-05, 3.822202779701911e-05, 3.2841395295690745e-05, 4.2974068492185324e-05, 4.391502443468198e-05, 3.842243313556537e-05, 3.7539692129939795e-05, 4.243620060151443e-05, 4.0751154301688075e-05, 4.494010136113502e-05, 4.0792532672639936e-05, 4.695989628089592e-05, 4.5364751713350415e-05, 4.3128704419359565e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00022091567916504573, "first_to_last_ratio": 1.232986382535498, "imbalance_abs_log_ratio": 0.20943917994887448, "last_quartile_mean": 0.00017917122386279516, "mean": 0.0002198539991695725, "normalized": [ 0.4672441326905094, 1.0299890566632133, 1.08986441571188, 1.0084939820045402, 0.8814237539069011, 1.2693826047962031, 1.1886514905905403, 1.1035827467935222, 1.0473307057033197, 1.4274044616309502, 1.1442998336482333, 1.0911283607583675, 1.1094038041384326, 1.1946873974128753, 1.2511466100570947, 0.9923165311956169, 0.9239703707694655, 1.2473578196113966, 1.1181336030529359, 1.0160745414007064, 0.9397394354798955, 0.9905446518023159, 1.0121913654482901, 0.9359943959598221, 0.8839446941554127, 0.8510229499852975, 0.8648513533654478, 0.8786898836618169, 0.7455559493639876, 0.8399702482950041, 0.7392957942269532, 0.7163130557190521 ], "peak_layer": 10, "peak_normalized": 1.4274044616309502, "population_cv": 0.18952709251819305, "rest_mean": 0.0002217885959436427, "spike_contrast": 0.9441746799407325, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00020940717658959329, "top_five_layers": [ 10, 6, 15, 18, 14 ], "top_five_spike_overlap": 0 }, "values": [ 0.00010272549116052687, 0.00022644721320830286, 0.00023961105034686625, 0.00022172143508214504, 0.0001937845372594893, 0.0002790788421407342, 0.0002613297838252038, 0.00024262708029709756, 0.00023025984410196543, 0.000313820579322055, 0.00025157889467664063, 0.00023988893372006714, 0.00024390686303377151, 0.000262656802078709, 0.00027506958576850593, 0.0002181647578254342, 0.00020313858112785965, 0.00027423660503700376, 0.0002458261442370713, 0.00022338805138133466, 0.00020660547306761146, 0.00021777520305477083, 0.00022253431961871684, 0.00020578211115207523, 0.0001943387760547921, 0.00018710079893935472, 0.0001901410287246108, 0.00019318348495289683, 0.00016391345707233995, 0.00018467081827111542, 0.00016253713693004102, 0.00015748428995721042 ] } } }, "pre_mlp_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "2999a257265024350fd2665e2a8d452a03cd662193b4245d15290408ad29922d", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.014061377383768559, 0.014165290631353855, 0.014281988143920898, 0.014350161887705326, 0.00966146495193243, 0.009741888381540775, 0.009820535778999329, 0.00991273857653141, 0.0074927168898284435, 0.007546746172010899, 0.00766373984515667, 0.007762390188872814, 0.006325984839349985, 0.006375366821885109, 0.006445344537496567, 0.006555322092026472, 0.005531130358576775, 0.005611324217170477, 0.0056777093559503555, 0.00574630219489336, 0.004946626722812653, 0.0050392793491482735, 0.005084683652967215, 0.005203457549214363, 0.004667261149734259, 0.004717131145298481, 0.004768412560224533, 0.004826710093766451, 0.0044052014127373695, 0.004504534415900707, 0.004574235528707504, 0.004644096828997135 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 3.729895206561196e-05, "first_to_last_ratio": 1.2469721805352876, "imbalance_abs_log_ratio": 0.22071835733540474, "last_quartile_mean": 2.9911615229138988e-05, "mean": 3.340667609563752e-05, "normalized": [ 1.308752778077389, 1.2456464596610868, 1.0236956931411572, 0.9922285647386262, 1.26903181062375, 1.1170846373362797, 1.0287316533628064, 0.9469237624012169, 1.1300261751959062, 1.1132698789554223, 1.0126240729566007, 0.9379202574439188, 1.1130503370514597, 1.070713488608195, 0.9897642722064988, 0.922085688720865, 1.036831181462568, 0.9898105546019274, 0.9395130252451869, 0.9001967292772246, 0.9975952535130206, 0.9404117749145336, 0.9224862220158919, 0.8885787590271026, 0.9461670180110204, 0.8974372095121397, 0.8904691498559152, 0.8684437934704528, 0.9244147277585869, 0.9015689206268677, 0.8740901912628669, 0.8604359589635168 ], "peak_layer": 1, "peak_normalized": 1.308752778077389, "population_cv": 0.1164905978879517, "rest_mean": 3.378375206213807e-05, "spike_contrast": 0.9285666618330386, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.137046587653458e-05, "top_five_layers": [ 1, 5, 2, 9, 6 ], "top_five_spike_overlap": 0 }, "values": [ 4.3721080146497115e-05, 4.161290780757554e-05, 3.419827044126578e-05, 3.314705827506259e-05, 4.239413465256803e-05, 3.73180846509058e-05, 3.4366505133220926e-05, 3.1633575417799875e-05, 3.7750418414361775e-05, 3.719064625329338e-05, 3.3828404411906376e-05, 3.1332798243965954e-05, 3.718331208801828e-05, 3.5768978705164045e-05, 3.306473445263691e-05, 3.080381793552078e-05, 3.463708344497718e-05, 3.3066280593629926e-05, 3.138600732199848e-05, 3.007258055731654e-05, 3.332634150865488e-05, 3.1416031561093405e-05, 3.0817198421573266e-05, 2.9684462788281962e-05, 3.1608295103069395e-05, 2.9980394174344838e-05, 2.9747614462394267e-05, 2.9011820515734144e-05, 3.0881623388268054e-05, 3.01184209092753e-05, 2.9200447897892445e-05, 2.874430538213346e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 6.705871965095866e-05, "first_to_last_ratio": 0.7180297289018083, "imbalance_abs_log_ratio": 0.33124430563536084, "last_quartile_mean": 9.33926785364747e-05, "mean": 8.228966771639534e-05, "normalized": [ 0.8685826408392191, 0.8249006353131848, 0.7154905339803773, 0.6884730939760333, 0.9470487095509537, 0.8638069620260335, 0.8256155016985178, 0.7853664463613442, 0.9851686390305121, 0.9692511243547932, 0.9083804947955202, 0.8729855457896263, 1.0422548534804916, 1.0255492616133721, 0.9871528463964067, 0.9398223255303129, 1.0958375595009802, 1.0726325633807163, 1.0495259774381736, 1.0230838788771643, 1.1576441025653552, 1.1142532603054807, 1.1008567438213033, 1.056908774400763, 1.159063578749664, 1.1192434432436902, 1.1032241584740294, 1.1060435702772593, 1.1786235078649367, 1.1628365878275773, 1.141840825041133, 1.1085318534950752 ], "peak_layer": 29, "peak_normalized": 1.1786235078649367, "population_cv": 0.1341844256762087, "rest_mean": 8.049536754023629e-05, "spike_contrast": 1.1426606459269575, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.197888866765424e-05, "top_five_layers": [ 29, 30, 25, 21, 31 ], "top_five_spike_overlap": 2 }, "values": [ 7.14753768988885e-05, 6.788079917896539e-05, 5.887747829547152e-05, 5.66542221349664e-05, 7.793232362018898e-05, 7.108238787623122e-05, 6.793962529627606e-05, 6.462754390668124e-05, 8.106919995043427e-05, 7.975935295689851e-05, 7.475032907677814e-05, 7.183769048424438e-05, 8.576680556870997e-05, 8.4392107964959e-05, 8.123247971525416e-05, 7.733766688033938e-05, 9.017610864248127e-05, 8.826657722238451e-05, 8.636514394311234e-05, 8.418923243880272e-05, 9.526214853394777e-05, 9.169153054244816e-05, 9.0589135652408e-05, 8.697267185198143e-05, 9.537895675748587e-05, 9.210217103827745e-05, 9.078394941752777e-05, 9.101595787797123e-05, 9.698853682493791e-05, 9.56894364207983e-05, 9.396170207764953e-05, 9.122071787714958e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.05946834245696664, "first_to_last_ratio": 0.7180297222365345, "imbalance_abs_log_ratio": 0.33124431491808765, "last_quartile_mean": 0.08282156102359295, "mean": 0.07297530013602227, "normalized": [ 0.8685826122359729, 0.8249005917343738, 0.71549052911611, 0.6884731350635278, 0.9470486699630107, 0.8638070113577097, 0.8256154769796886, 0.7853664269052657, 0.9851686341386658, 0.9692511556795396, 0.9083805454214465, 0.8729855579949275, 1.0422548040385637, 1.0255492365750694, 0.9871528950273681, 0.9398223256645052, 1.0958375053304956, 1.07263252629438, 1.0495259690486791, 1.0230837911446407, 1.1576441435704576, 1.1142533045483842, 1.1008568124470575, 1.0569088284972943, 1.1590636022344982, 1.119243411839159, 1.1032241423789504, 1.1060435591240758, 1.1786235037173105, 1.1628366076958818, 1.141840809884862, 1.1085318743481278 ], "peak_layer": 29, "peak_normalized": 1.1786235037173105, "population_cv": 0.13418442668798974, "rest_mean": 0.07138409617322462, "spike_contrast": 1.1426607032635474, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.08156780153512955, "top_five_layers": [ 29, 30, 25, 21, 31 ], "top_five_spike_overlap": 2 }, "values": [ 0.06338507682085037, 0.0601973682641983, 0.05221313610672951, 0.050241533666849136, 0.0691111609339714, 0.06303657591342926, 0.060249537229537964, 0.05731235072016716, 0.07189297676086426, 0.07073139399290085, 0.06628934293985367, 0.06370638310909271, 0.07605885714292526, 0.0748397633433342, 0.07203777879476547, 0.06858381628990173, 0.07996907085180283, 0.07827568054199219, 0.07658947259187698, 0.07465984672307968, 0.0844794288277626, 0.08131296932697296, 0.0803353562951088, 0.07712823897600174, 0.08458301424980164, 0.08167712390422821, 0.08050811290740967, 0.08071386069059372, 0.0860104039311409, 0.08485835045576096, 0.08332617580890656, 0.08089544624090195 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 7.408341275549901e-06, "first_to_last_ratio": 0.49124437919492936, "imbalance_abs_log_ratio": 0.7108135577065074, "last_quartile_mean": 1.5080765479069669e-05, "mean": 1.1844782648040564e-05, "normalized": [ 0.5657990507718705, 0.5382233871697658, 0.49942145823009426, 0.48906984415931193, 0.8062921756393574, 0.7122953827323945, 0.7013508394120195, 0.6911626993599347, 0.8940675881372303, 0.868722127857376, 0.9735911331661544, 0.854855860776803, 0.989483587379368, 1.0059088486455374, 1.0303729006301217, 0.9941025538400318, 1.1620912657632405, 1.127012997009919, 1.113401270263767, 1.066097770938846, 1.1826034542812638, 1.1643547935214302, 1.2049778184223279, 1.1791491538213539, 1.2365172470752577, 1.219921447624247, 1.2337761202582145, 1.2288278254433027, 1.3039217543779833, 1.3631114810840155, 1.2856635723507386, 1.3138525898567213 ], "peak_layer": 30, "peak_normalized": 1.3631114810840155, "population_cv": 0.2549101540724733, "rest_mean": 1.1420299651271543e-05, "spike_contrast": 1.237882653019464, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.4136990830593277e-05, "top_five_layers": [ 30, 32, 29, 31, 25 ], "top_five_spike_overlap": 1 }, "values": [ 6.701766778860474e-06, 6.3751390371180605e-06, 5.915538622502936e-06, 5.7929260037781205e-06, 9.550355571263935e-06, 8.436983989668079e-06, 8.307348252856173e-06, 8.186671948351432e-06, 1.0590036254143342e-05, 1.0289824786013924e-05, 1.1531975360412616e-05, 1.0125581866304856e-05, 1.1720218026312068e-05, 1.1914771675947122e-05, 1.220454305439489e-05, 1.1774928680097219e-05, 1.3764718460151926e-05, 1.334922399109928e-05, 1.3187996046326589e-05, 1.2627696378331166e-05, 1.4007680874783546e-05, 1.379152945446549e-05, 1.4272700354922563e-05, 1.3966765436634887e-05, 1.4646278032159898e-05, 1.4449704394792207e-05, 1.4613809980801307e-05, 1.4555198504240252e-05, 1.5444669770658948e-05, 1.614575921848882e-05, 1.5228405572997872e-05, 1.5562298358418047e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 5.6029601637419546e-05, "first_to_last_ratio": 0.6070213107493427, "imbalance_abs_log_ratio": 0.49919138022039955, "last_quartile_mean": 9.230252817360451e-05, "mean": 7.840498903988191e-05, "normalized": [ 0.6598133090619259, 0.6419677920829977, 0.6005256373772865, 0.5913463573822231, 0.8562376513989928, 0.8062968559250673, 0.7923156566726156, 0.768439227921366, 0.9790122729268586, 0.9665730330981341, 0.9163710065350301, 0.8881686280668529, 1.0631190324760253, 1.0511030519715439, 1.0188080221730096, 0.9712812246823179, 1.1361272291219966, 1.1138561419734825, 1.0918134348286874, 1.0639217496772315, 1.203414696914147, 1.1592426077267617, 1.1442664103435203, 1.097952968622282, 1.2042281788778202, 1.1625445130141736, 1.1452986211445626, 1.1473932954411665, 1.2233942816523278, 1.205318760675343, 1.1829469858778916, 1.146901364356358 ], "peak_layer": 29, "peak_normalized": 1.2233942816523278, "population_cv": 0.1898597249088972, "rest_mean": 7.605543874820728e-05, "spike_contrast": 1.1977126437531995, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.109256061492487e-05, "top_five_layers": [ 29, 30, 25, 21, 31 ], "top_five_spike_overlap": 2 }, "values": [ 5.173265526536852e-05, 5.033347770222463e-05, 4.708420601673424e-05, 4.636450466932729e-05, 6.713330367347226e-05, 6.321769615169615e-05, 6.212150037754327e-05, 6.024946924299002e-05, 7.675944652874023e-05, 7.578414806630462e-05, 7.184805872384459e-05, 6.963685154914856e-05, 8.335383608937263e-05, 8.241172326961532e-05, 7.987963181221858e-05, 7.615329377586022e-05, 8.907804294722155e-05, 8.733187860343605e-05, 8.560362039133906e-05, 8.341677312273532e-05, 9.435371612198651e-05, 9.089040395338088e-05, 8.971619536168873e-05, 8.608499047113582e-05, 9.441749716643244e-05, 9.114928980125114e-05, 8.97971258382313e-05, 8.996135875349864e-05, 9.592021524440497e-05, 9.450300422031432e-05, 9.274894546251744e-05, 8.992278890218586e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 4.8511469685763586e-05, "first_to_last_ratio": 0.5271352332510719, "imbalance_abs_log_ratio": 0.6402981537748061, "last_quartile_mean": 9.202850924339145e-05, "mean": 7.532146378252946e-05, "normalized": [ 0.548104082267993, 0.5416830199005969, 0.528884702689083, 0.5247448675895723, 0.7777813723908718, 0.7487613756360456, 0.7496059866168319, 0.7329069161583202, 0.9480676391902623, 0.9399123881532756, 0.9000936195631334, 0.883441930780847, 1.0560589316963704, 1.0534552095740957, 1.032095781768693, 0.9835620691098637, 1.1601754560133901, 1.1467788545782247, 1.1233340534265408, 1.0967375255670042, 1.2396083970871608, 1.1951467072205837, 1.1828066060767042, 1.1317734013234224, 1.244698956870482, 1.2025862582796845, 1.1881970083126314, 1.1882256015362047, 1.2689460104605979, 1.2589162611011824, 1.2307315535498626, 1.1921774555104672 ], "peak_layer": 29, "peak_normalized": 1.2689460104605979, "population_cv": 0.23486334953575583, "rest_mean": 7.254842300123225e-05, "spike_contrast": 1.244629176846487, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.029588400153444e-05, "top_five_layers": [ 29, 30, 25, 21, 31 ], "top_five_spike_overlap": 2 }, "values": [ 4.1284001781605184e-05, 4.080035796505399e-05, 3.9836369978729635e-05, 3.952455153921619e-05, 5.858363147126511e-05, 5.639780283672735e-05, 5.6461420172126964e-05, 5.520362174138427e-05, 7.140984234865755e-05, 7.079557690303773e-05, 6.77963689668104e-05, 6.654213939327747e-05, 7.954390457598493e-05, 7.934778841445222e-05, 7.773896504659206e-05, 7.408333476632833e-05, 8.738611359149218e-05, 8.637706196168438e-05, 8.461116522084922e-05, 8.26078758109361e-05, 9.336911898571998e-05, 9.002019942272455e-05, 8.909072494134307e-05, 8.524682925781235e-05, 9.375254740007222e-05, 9.058055729838088e-05, 8.949673792812973e-05, 8.949889161158353e-05, 9.557887096889317e-05, 9.482341556577012e-05, 9.270050213672221e-05, 8.979655103757977e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 9.61669284151867e-05, "first_to_last_ratio": 0.8297833624837898, "imbalance_abs_log_ratio": 0.18659062131596024, "last_quartile_mean": 0.00011589401856326731, "mean": 0.00010716335873439675, "normalized": [ 0.9380742209455786, 0.8898820858744699, 0.7837417114644826, 0.7583102664636657, 1.0822070619439073, 0.9520749081713696, 0.9107982467806989, 0.8640017807748708, 1.0669538303069324, 1.0266950649381863, 0.9594029855154438, 0.8901506143407661, 1.0795289063552718, 1.0379165674077444, 0.9706971366471856, 0.914979822300863, 1.052939971267624, 1.0225707935964177, 0.9851238030379684, 0.9636652893153618, 1.0990044510799908, 1.0523904893769196, 1.0422234788441622, 1.0049019599938585, 1.0962247910521359, 1.0599378706264646, 1.05039930496957, 1.0583562345651827, 1.1199788667078585, 1.1100529546792268, 1.0850973923636869, 1.0717171382921347 ], "peak_layer": 29, "peak_normalized": 1.1199788667078585, "population_cv": 0.09156717959150794, "rest_mean": 0.00010599351123716958, "spike_contrast": 1.0706366257223145, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00011348053521942347, "top_five_layers": [ 29, 30, 21, 25, 31 ], "top_five_spike_overlap": 2 }, "values": [ 0.00010052718425868079, 9.536275319987908e-05, 8.398839418077841e-05, 8.12630751170218e-05, 0.00011597294360399246, 0.00010202754492638633, 9.760419925441965e-05, 9.25893327803351e-05, 0.00011433835607022047, 0.00011002409155480564, 0.00010281284630764276, 9.539152961224318e-05, 0.000115685943455901, 0.0001112266254494898, 0.0001040231654769741, 9.805231093196198e-05, 0.00011283658386673778, 0.00010958212078548968, 0.00010556917550275102, 0.00010326960909878835, 0.00011777300824178383, 0.0001127776995417662, 0.00011168816854478791, 0.00010768866923172027, 0.00011747513053705916, 0.00011358650226611644, 0.00011256431753281504, 0.00011341700883349404, 0.00012002069706795737, 0.00011895700299646705, 0.00011628268111962825, 0.00011484880815260112 ] } } } }, "selector": { "expected_uniform_count": 57, "identities_unique": true, "identity_order_sha256": "97fd9721a263024f6a746638f32e0ad9a988cd4c111c00e9e178c1022e5c9d5e", "passed": true, "uniform_count": 57, "uniform_indices": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64 ], "visit_count": 65, "visits": [ { "branch": "attention", "coefficient": "uniform", "group": 1, "index": 0, "kind": "depth", "layer": 1, "offset": 1, "sources": 1, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 1, "index": 1, "kind": "depth", "layer": 1, "offset": 1, "sources": 2, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 1, "index": 2, "kind": "depth", "layer": 2, "offset": 2, "sources": 2, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 1, "index": 3, "kind": "depth", "layer": 2, "offset": 2, "sources": 2, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 1, "index": 4, "kind": "depth", "layer": 3, "offset": 3, "sources": 2, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 1, "index": 5, "kind": "depth", "layer": 3, "offset": 3, "sources": 2, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 1, "index": 6, "kind": "depth", "layer": 4, "offset": 4, "sources": 2, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 1, "index": 7, "kind": "depth", "layer": 4, "offset": 4, "sources": 2, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 2, "index": 8, "kind": "depth", "layer": 5, "offset": 1, "sources": 2, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 2, "index": 9, "kind": "depth", "layer": 5, "offset": 1, "sources": 3, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 2, "index": 10, "kind": "depth", "layer": 6, "offset": 2, "sources": 3, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 2, "index": 11, "kind": "depth", "layer": 6, "offset": 2, "sources": 3, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 2, "index": 12, "kind": "depth", "layer": 7, "offset": 3, "sources": 3, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 2, "index": 13, "kind": "depth", "layer": 7, "offset": 3, "sources": 3, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 2, "index": 14, "kind": "depth", "layer": 8, "offset": 4, "sources": 3, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 2, "index": 15, "kind": "depth", "layer": 8, "offset": 4, "sources": 3, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 3, "index": 16, "kind": "depth", "layer": 9, "offset": 1, "sources": 3, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 3, "index": 17, "kind": "depth", "layer": 9, "offset": 1, "sources": 4, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 3, "index": 18, "kind": "depth", "layer": 10, "offset": 2, "sources": 4, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 3, "index": 19, "kind": "depth", "layer": 10, "offset": 2, "sources": 4, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 3, "index": 20, "kind": "depth", "layer": 11, "offset": 3, "sources": 4, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 3, "index": 21, "kind": "depth", "layer": 11, "offset": 3, "sources": 4, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 3, "index": 22, "kind": "depth", "layer": 12, "offset": 4, "sources": 4, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 3, "index": 23, "kind": "depth", "layer": 12, "offset": 4, "sources": 4, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 4, "index": 24, "kind": "depth", "layer": 13, "offset": 1, "sources": 4, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 4, "index": 25, "kind": "depth", "layer": 13, "offset": 1, "sources": 5, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 4, "index": 26, "kind": "depth", "layer": 14, "offset": 2, "sources": 5, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 4, "index": 27, "kind": "depth", "layer": 14, "offset": 2, "sources": 5, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 4, "index": 28, "kind": "depth", "layer": 15, "offset": 3, "sources": 5, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 4, "index": 29, "kind": "depth", "layer": 15, "offset": 3, "sources": 5, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 4, "index": 30, "kind": "depth", "layer": 16, "offset": 4, "sources": 5, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 4, "index": 31, "kind": "depth", "layer": 16, "offset": 4, "sources": 5, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 5, "index": 32, "kind": "depth", "layer": 17, "offset": 1, "sources": 5, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 5, "index": 33, "kind": "depth", "layer": 17, "offset": 1, "sources": 6, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 5, "index": 34, "kind": "depth", "layer": 18, "offset": 2, "sources": 6, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 5, "index": 35, "kind": "depth", "layer": 18, "offset": 2, "sources": 6, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 5, "index": 36, "kind": "depth", "layer": 19, "offset": 3, "sources": 6, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 5, "index": 37, "kind": "depth", "layer": 19, "offset": 3, "sources": 6, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 5, "index": 38, "kind": "depth", "layer": 20, "offset": 4, "sources": 6, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 5, "index": 39, "kind": "depth", "layer": 20, "offset": 4, "sources": 6, "use_uniform": true }, { "branch": "attention", "coefficient": "detached_learned", "group": 6, "index": 40, "kind": "depth", "layer": 21, "offset": 1, "sources": 6, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 6, "index": 41, "kind": "depth", "layer": 21, "offset": 1, "sources": 7, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 6, "index": 42, "kind": "depth", "layer": 22, "offset": 2, "sources": 7, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 6, "index": 43, "kind": "depth", "layer": 22, "offset": 2, "sources": 7, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 6, "index": 44, "kind": "depth", "layer": 23, "offset": 3, "sources": 7, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 6, "index": 45, "kind": "depth", "layer": 23, "offset": 3, "sources": 7, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 6, "index": 46, "kind": "depth", "layer": 24, "offset": 4, "sources": 7, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 6, "index": 47, "kind": "depth", "layer": 24, "offset": 4, "sources": 7, "use_uniform": false }, { "branch": "attention", "coefficient": "uniform", "group": 7, "index": 48, "kind": "depth", "layer": 25, "offset": 1, "sources": 7, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 7, "index": 49, "kind": "depth", "layer": 25, "offset": 1, "sources": 8, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 7, "index": 50, "kind": "depth", "layer": 26, "offset": 2, "sources": 8, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 7, "index": 51, "kind": "depth", "layer": 26, "offset": 2, "sources": 8, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 7, "index": 52, "kind": "depth", "layer": 27, "offset": 3, "sources": 8, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 7, "index": 53, "kind": "depth", "layer": 27, "offset": 3, "sources": 8, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 7, "index": 54, "kind": "depth", "layer": 28, "offset": 4, "sources": 8, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 7, "index": 55, "kind": "depth", "layer": 28, "offset": 4, "sources": 8, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 8, "index": 56, "kind": "depth", "layer": 29, "offset": 1, "sources": 8, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 8, "index": 57, "kind": "depth", "layer": 29, "offset": 1, "sources": 9, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 8, "index": 58, "kind": "depth", "layer": 30, "offset": 2, "sources": 9, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 8, "index": 59, "kind": "depth", "layer": 30, "offset": 2, "sources": 9, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 8, "index": 60, "kind": "depth", "layer": 31, "offset": 3, "sources": 9, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 8, "index": 61, "kind": "depth", "layer": 31, "offset": 3, "sources": 9, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 8, "index": 62, "kind": "depth", "layer": 32, "offset": 4, "sources": 9, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 8, "index": 63, "kind": "depth", "layer": 32, "offset": 4, "sources": 9, "use_uniform": true }, { "branch": "output", "coefficient": "uniform", "group": null, "index": 64, "kind": "output", "layer": null, "offset": null, "sources": 9, "use_uniform": true } ], "visits_sha256": "9b343b963f66443eda485b5123dbb230016e3c1d527af25d484ae56d6fa0c934" } }, "uniform_except_group_7": { "bits_per_byte": 7.959341947457508, "forward": { "activation_sha256": { "attention_branch_output": "676759a2d7c0615ad7e1f246e90927fb829d00c8b6c3745032ae6b27ab7d902a", "mlp_branch_output": "afe29f3ef4d2992353f0bb59fe7c5c2c5050fca671a53184825a8d63cd83a2bc", "post_attention_state": "31853ffb07b24cc59e774bb20656840b4358b3080b42516d380f2dc0f46404c8", "post_mlp_state": "7fd9c911e93175896ad618842c120e77b3f017c17b7eec9c2bfb01fc6dbc8ac2", "pre_attention_input": "de71d4cf2d509a1063f6eac7f50c4885e9a058dee7e4b993ad40f5527f528d00", "pre_mlp_input": "2999a257265024350fd2665e2a8d452a03cd662193b4245d15290408ad29922d" }, "logits_sha256": "873a309a197bfd08fa523edcb440d2697903a34abc02dad1c7f5991509c3c151", "loss_nats": 5.516995429992676, "mixer_summary_sha256": "8450b24bdc777885a3357dc7b8080e7410acbe96943686433da76f379295133a" }, "loss_nats": 5.516995429992676, "loss_scale": 1.0, "mixers": [ { "branch": "attention", "entropy_mean": -0.0, "group": 1, "latest_source_mass": 1.0, "layer": 1, "max_source_mass": 1.0, "mixer_index": 0, "normalized_entropy": 1.0, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 1.0, "median": 1.0, "p05": 1.0, "p95": 1.0 } ], "sources": 1 }, { "branch": "mlp", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 1, "max_source_mass": 0.5, "mixer_index": 1, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 2, "max_source_mass": 0.5, "mixer_index": 2, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 2, "max_source_mass": 0.5, "mixer_index": 3, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 3, "max_source_mass": 0.5, "mixer_index": 4, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 3, "max_source_mass": 0.5, "mixer_index": 5, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 4, "max_source_mass": 0.5, "mixer_index": 6, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 4, "max_source_mass": 0.5, "mixer_index": 7, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6931471824645996, "group": 2, "latest_source_mass": 0.5, "layer": 5, "max_source_mass": 0.5, "mixer_index": 8, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "completed_group_1", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 5, "max_source_mass": 0.3333333432674408, "mixer_index": 9, "normalized_entropy": 1.0000000180537645, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 6, "max_source_mass": 0.3333333432674408, "mixer_index": 10, "normalized_entropy": 1.0000000180537645, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 6, "max_source_mass": 0.3333333432674408, "mixer_index": 11, "normalized_entropy": 1.0000000180537645, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 7, "max_source_mass": 0.3333333432674408, "mixer_index": 12, "normalized_entropy": 1.0000000180537645, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 7, "max_source_mass": 0.3333333432674408, "mixer_index": 13, "normalized_entropy": 1.0000000180537645, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 8, "max_source_mass": 0.3333333432674408, "mixer_index": 14, "normalized_entropy": 1.0000000180537645, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 8, "max_source_mass": 0.3333333432674408, "mixer_index": 15, "normalized_entropy": 1.0000000180537645, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0986123085021973, "group": 3, "latest_source_mass": 0.3333333432674408, "layer": 9, "max_source_mass": 0.3333333432674408, "mixer_index": 16, "normalized_entropy": 1.0000000180537645, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_2", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 9, "max_source_mass": 0.25, "mixer_index": 17, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 10, "max_source_mass": 0.25, "mixer_index": 18, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 10, "max_source_mass": 0.25, "mixer_index": 19, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 11, "max_source_mass": 0.25, "mixer_index": 20, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 11, "max_source_mass": 0.25, "mixer_index": 21, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 12, "max_source_mass": 0.25, "mixer_index": 22, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 12, "max_source_mass": 0.25, "mixer_index": 23, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3862943649291992, "group": 4, "latest_source_mass": 0.25, "layer": 13, "max_source_mass": 0.25, "mixer_index": 24, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_3", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 13, "max_source_mass": 0.20000000298023224, "mixer_index": 25, "normalized_entropy": 1.0000000186840277, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 14, "max_source_mass": 0.20000000298023224, "mixer_index": 26, "normalized_entropy": 1.0000000186840277, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 14, "max_source_mass": 0.20000000298023224, "mixer_index": 27, "normalized_entropy": 1.0000000186840277, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 15, "max_source_mass": 0.20000000298023224, "mixer_index": 28, "normalized_entropy": 1.0000000186840277, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 15, "max_source_mass": 0.20000000298023224, "mixer_index": 29, "normalized_entropy": 1.0000000186840277, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 16, "max_source_mass": 0.20000000298023224, "mixer_index": 30, "normalized_entropy": 1.0000000186840277, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 16, "max_source_mass": 0.20000000298023224, "mixer_index": 31, "normalized_entropy": 1.0000000186840277, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.6094379425048828, "group": 5, "latest_source_mass": 0.20000000298023224, "layer": 17, "max_source_mass": 0.20000000298023224, "mixer_index": 32, "normalized_entropy": 1.0000000186840277, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_4", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 17, "max_source_mass": 0.1666666716337204, "mixer_index": 33, "normalized_entropy": 1.0000001451965654, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 18, "max_source_mass": 0.1666666716337204, "mixer_index": 34, "normalized_entropy": 1.0000001451965654, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 18, "max_source_mass": 0.1666666716337204, "mixer_index": 35, "normalized_entropy": 1.0000001451965654, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 19, "max_source_mass": 0.1666666716337204, "mixer_index": 36, "normalized_entropy": 1.0000001451965654, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 19, "max_source_mass": 0.1666666716337204, "mixer_index": 37, "normalized_entropy": 1.0000001451965654, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 20, "max_source_mass": 0.1666666716337204, "mixer_index": 38, "normalized_entropy": 1.0000001451965654, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 20, "max_source_mass": 0.1666666716337204, "mixer_index": 39, "normalized_entropy": 1.0000001451965654, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.791759729385376, "group": 6, "latest_source_mass": 0.1666666716337204, "layer": 21, "max_source_mass": 0.1666666716337204, "mixer_index": 40, "normalized_entropy": 1.0000001451965654, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_5", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 21, "max_source_mass": 0.1428571492433548, "mixer_index": 41, "normalized_entropy": 1.00000003408302, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 22, "max_source_mass": 0.1428571492433548, "mixer_index": 42, "normalized_entropy": 1.00000003408302, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 22, "max_source_mass": 0.1428571492433548, "mixer_index": 43, "normalized_entropy": 1.00000003408302, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 23, "max_source_mass": 0.1428571492433548, "mixer_index": 44, "normalized_entropy": 1.00000003408302, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 23, "max_source_mass": 0.1428571492433548, "mixer_index": 45, "normalized_entropy": 1.00000003408302, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 24, "max_source_mass": 0.1428571492433548, "mixer_index": 46, "normalized_entropy": 1.00000003408302, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 24, "max_source_mass": 0.1428571492433548, "mixer_index": 47, "normalized_entropy": 1.00000003408302, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9459102153778076, "group": 7, "latest_source_mass": 0.1428571492433548, "layer": 25, "max_source_mass": 0.1428571492433548, "mixer_index": 48, "normalized_entropy": 1.00000003408302, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_6", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 25, "max_source_mass": 0.125, "mixer_index": 49, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 26, "max_source_mass": 0.125, "mixer_index": 50, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 26, "max_source_mass": 0.125, "mixer_index": 51, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 27, "max_source_mass": 0.125, "mixer_index": 52, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 27, "max_source_mass": 0.125, "mixer_index": 53, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 28, "max_source_mass": 0.125, "mixer_index": 54, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 28, "max_source_mass": 0.125, "mixer_index": 55, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.079441547393799, "group": 8, "latest_source_mass": 0.125, "layer": 29, "max_source_mass": 0.125, "mixer_index": 56, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_7", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 29, "max_source_mass": 0.1111111119389534, "mixer_index": 57, "normalized_entropy": 1.0000000180537645, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 30, "max_source_mass": 0.1111111119389534, "mixer_index": 58, "normalized_entropy": 1.0000000180537645, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 30, "max_source_mass": 0.1111111119389534, "mixer_index": 59, "normalized_entropy": 1.0000000180537645, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 31, "max_source_mass": 0.1111111119389534, "mixer_index": 60, "normalized_entropy": 1.0000000180537645, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 31, "max_source_mass": 0.1111111119389534, "mixer_index": 61, "normalized_entropy": 1.0000000180537645, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 32, "max_source_mass": 0.1111111119389534, "mixer_index": 62, "normalized_entropy": 1.0000000180537645, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 32, "max_source_mass": 0.1111111119389534, "mixer_index": 63, "normalized_entropy": 1.0000000180537645, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "output", "entropy_mean": 2.1972246170043945, "group": null, "latest_source_mass": 0.1111111119389534, "layer": null, "max_source_mass": 0.1111111119389534, "mixer_index": null, "normalized_entropy": 1.0000000180537645, "offset": null, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_8", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 } ], "mode": "uniform_except_group_7", "optimizer_state_after": "b82a349700eda58e2330a22ca2ee1f7fb1406efd47fd9aa306122b368a114192", "optimizer_state_before": "b82a349700eda58e2330a22ca2ee1f7fb1406efd47fd9aa306122b368a114192", "optimizer_state_unchanged": true, "positions": { "attention_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "676759a2d7c0615ad7e1f246e90927fb829d00c8b6c3745032ae6b27ab7d902a", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.0022265110164880753, 0.0021613112185150385, 0.0025601957459002733, 0.0025323182344436646, 0.0027253651060163975, 0.002904455177485943, 0.0029208383057266474, 0.00299624796025455, 0.0030937574338167906, 0.0033804215490818024, 0.0034370317589491606, 0.0038969130255281925, 0.004255065228790045, 0.004119740333408117, 0.003970506135374308, 0.004385598469525576, 0.0041999719105660915, 0.005232699681073427, 0.004664790350943804, 0.004898969549685717, 0.005131533369421959, 0.005069608334451914, 0.0051318020559847355, 0.0058372304774820805, 0.0059690922498703, 0.00554921617731452, 0.0058550527319312096, 0.006255472078919411, 0.006502739619463682, 0.006122888531535864, 0.007071305997669697, 0.006241494789719582 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.00015962477027642308, "first_to_last_ratio": 3.7699040072151497, "imbalance_abs_log_ratio": 1.3270495388586014, "last_quartile_mean": 4.234186599205714e-05, "mean": 8.617327489446325e-05, "normalized": [ 2.5428430303860874, 2.3222064178412967, 2.1011513502014654, 1.8911470095267005, 1.7004447966680725, 1.5353700031893112, 1.4229335846259188, 1.3028620492588128, 1.2049362956920768, 1.1705197150696323, 1.0773612761873523, 1.0133200843661747, 0.9906112963967714, 0.9272530070793641, 0.8550207003106389, 0.8185600560490035, 0.7703816559177782, 0.7173918665339755, 0.6760157258502337, 0.6514535263006792, 0.6184138115448249, 0.6053606047045522, 0.5838821473278404, 0.5697017460729616, 0.5457761369570343, 0.5276086337508958, 0.5104655265174319, 0.4951752820428687, 0.47527333481689216, 0.465789239220051, 0.45896414036514044, 0.45180594922816086 ], "peak_layer": 1, "peak_normalized": 2.5428430303860874, "population_cv": 0.5705341151299013, "rest_mean": 9.280180475562466e-05, "spike_contrast": 0.5428688997682254, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.0379213644191625e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00021912511147093028, 0.00020011213200632483, 0.00018106309289578348, 0.00016296633111778647, 0.00014653289690613747, 0.00013230786134954542, 0.0001226188469445333, 0.00011227188952034339, 0.0001038333066389896, 0.00010086751717608422, 9.283974941354245e-05, 8.732111018616706e-05, 8.536421955795959e-05, 7.990442827576771e-05, 7.367993384832516e-05, 7.053800072753802e-05, 6.63863102090545e-05, 6.182000652188435e-05, 5.825448897667229e-05, 5.6137883802875876e-05, 5.329074338078499e-05, 5.216590579948388e-05, 5.031503678765148e-05, 4.909306517220102e-05, 4.703131708083674e-05, 4.5465763832908124e-05, 4.3988486140733585e-05, 4.2670875700423494e-05, 4.095585973118432e-05, 4.013858415419236e-05, 3.955044303438626e-05, 3.893359826179221e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.00021442661636683624, "first_to_last_ratio": 1.843144421770586, "imbalance_abs_log_ratio": 0.6114730379455638, "last_quartile_mean": 0.0001163373926829081, "mean": 0.00015221274247778638, "normalized": [ 1.7741446897870732, 1.648461275666186, 1.5301464224684427, 1.4217121808823485, 1.3265065496562456, 1.2454420544341354, 1.1897998831173078, 1.1336248235930768, 1.0869089599614004, 1.0643003125599004, 1.0227030055526172, 0.9941765648180964, 0.9788958472869692, 0.9511102301475627, 0.9228060651723192, 0.9052418824754537, 0.8846484396889256, 0.865292475366091, 0.8497311645646503, 0.8388691935254817, 0.8256303537004397, 0.8176325372729673, 0.8076852907347736, 0.8000670639426607, 0.7892588692968658, 0.7814415503461742, 0.7731202151361662, 0.7664994567889658, 0.758490072461763, 0.7529625284402992, 0.7485174912671357, 0.7441725498875049 ], "peak_layer": 1, "peak_normalized": 1.7741446897870732, "population_cv": 0.27364872432853743, "rest_mean": 0.00015762320577373935, "spike_contrast": 0.7803181014868814, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012299624067964033, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00027004742878489196, 0.0002509168116375804, 0.0002329077833564952, 0.00021640271006617695, 0.00020191119983792305, 0.00018957215070258826, 0.0001811027032090351, 0.00017255214333999902, 0.00016544139361940324, 0.00016200006939470768, 0.00015566842921543866, 0.0001513263414381072, 0.00014900042151566595, 0.0001447710965294391, 0.00014046284195501357, 0.0001377893495373428, 0.00013465476513374597, 0.00013170854072086513, 0.00012933991092722863, 0.0001276865805266425, 0.00012567146040964872, 0.00012445409083738923, 0.00012293999316170812, 0.00012178040196886286, 0.00012013525702059269, 0.00011894536146428436, 0.00011767874821089208, 0.00011667098442558199, 0.00011545185407157987, 0.00011461049143690616, 0.00011393390013836324, 0.00011327254469506443 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.19015566632151604, "first_to_last_ratio": 1.8431444167877922, "imbalance_abs_log_ratio": 0.6114730352421437, "last_quartile_mean": 0.10316916275769472, "mean": 0.13498378219082952, "normalized": [ 1.7741447141792424, 1.6484611693507762, 1.5301464164964773, 1.421712099711509, 1.3265065258282553, 1.245442086656075, 1.1897999853143832, 1.1336247998774232, 1.0869090145314644, 1.0643003539468234, 1.0227030148964142, 0.9941765578539046, 0.978895844117059, 0.9511102293531465, 0.9228060470726628, 0.9052419251832852, 0.8846484748200404, 0.8652925213936268, 0.8497311905993837, 0.8388692019864461, 0.8256303029992552, 0.8176324960789627, 0.8076852727084025, 0.800067049482784, 0.789258877083647, 0.7814415614650243, 0.7731201949085273, 0.7664994759042474, 0.758490077799444, 0.7529625178867759, 0.7485174642151933, 0.7441725362993381 ], "peak_layer": 1, "peak_normalized": 1.7741447141792424, "population_cv": 0.2736487184871982, "rest_mean": 0.13978183573042904, "spike_contrast": 0.7803180757143806, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.10907429307699204, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.23948076367378235, 0.2225155234336853, 0.20654495060443878, 0.1919080764055252, 0.17905686795711517, 0.16811448335647583, 0.16060370206832886, 0.1530209630727768, 0.14671508967876434, 0.14366328716278076, 0.13804832100868225, 0.13419771194458008, 0.1321350634098053, 0.12838445603847504, 0.12456385046243668, 0.12219297885894775, 0.11941319704055786, 0.116800457239151, 0.11469992995262146, 0.11323373764753342, 0.11144670099020004, 0.11036712676286697, 0.10902441293001175, 0.10799607634544373, 0.10653714835643768, 0.10548193752765656, 0.10435868799686432, 0.1034649983048439, 0.10238385945558548, 0.10163772851228714, 0.10103771835565567, 0.10045122355222702 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 6.883365585963475e-05, "first_to_last_ratio": 2.2497546817252747, "imbalance_abs_log_ratio": 0.8108211799277664, "last_quartile_mean": 3.059607183786284e-05, "mean": 4.691841348858361e-05, "normalized": [ 1.7359303646705315, 1.6475859007944922, 1.5536416135247637, 1.4921074478374423, 1.436565604934822, 1.343834173349448, 1.2787439414688093, 1.2483315192759759, 1.191227587726315, 1.1603086071531024, 1.0995306735006558, 1.059977946939621, 1.0583984121521266, 1.0176934649371145, 0.9754909410255406, 0.9631621800279858, 0.9242301483750994, 0.8752327792179452, 0.8454160845678106, 0.8232909667825862, 0.7863453139125789, 0.77746685672799, 0.7497386589361914, 0.7388508716017261, 0.7093180450413769, 0.6908354487677191, 0.6685471901846501, 0.6592354484095886, 0.631440334976849, 0.6235901145414114, 0.6199506557967245, 0.6139807028410055 ], "peak_layer": 1, "peak_normalized": 1.7359303646705315, "population_cv": 0.3220412433027962, "rest_mean": 4.907019663530971e-05, "spike_contrast": 0.7193528234378936, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.529878449626267e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 8.144709863699973e-05, 7.730211655143648e-05, 7.289439963642508e-05, 7.000731420703232e-05, 6.740137905580923e-05, 6.305056740529835e-05, 5.999663699185476e-05, 5.856973439222202e-05, 5.5890508519951254e-05, 5.4439839004771784e-05, 5.1588234782684594e-05, 4.9732483603293076e-05, 4.965837433701381e-05, 4.774856279254891e-05, 4.576848732540384e-05, 4.519004141911864e-05, 4.3363412260077894e-05, 4.106453343410976e-05, 3.966558142565191e-05, 3.862750600092113e-05, 3.6894074582960457e-05, 3.647751145763323e-05, 3.517654840834439e-05, 3.4665710700210184e-05, 3.328007733216509e-05, 3.241290323785506e-05, 3.136717350571416e-05, 3.0930281354812905e-05, 2.9626178729813546e-05, 2.925785884144716e-05, 2.9087101211189292e-05, 2.8807000489905477e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 0.0001734125216898974, "first_to_last_ratio": 1.495517223041696, "imbalance_abs_log_ratio": 0.4024721155990803, "last_quartile_mean": 0.00011595488103921525, "mean": 0.00013990532806928968, "normalized": [ 1.3768889945483427, 1.3340940553978193, 1.2910049686488183, 1.2527203273458163, 1.2147536525239127, 1.1769854337196524, 1.14744939519443, 1.1220955988146075, 1.0944022472959734, 1.0773022658134643, 1.0508559212107247, 1.03206209473486, 1.0208171898483585, 1.000561882500425, 0.9805161605170362, 0.9663129296065233, 0.9487424991891081, 0.931931984738623, 0.9181974345490227, 0.9075104531838204, 0.8943640941785878, 0.8862664021756573, 0.8757830774574916, 0.86790401991425, 0.8562894022841879, 0.8478155487572495, 0.8387227683262297, 0.8314011142413483, 0.8224071457687988, 0.8161875050523302, 0.8112199678685613, 0.8064334645939688 ], "peak_layer": 1, "peak_normalized": 1.3768889945483427, "population_cv": 0.16325957486062354, "rest_mean": 0.0001431148236776978, "spike_contrast": 0.8564734849544942, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001225740517838858, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0001926341064972803, 0.00018664686649572104, 0.00018061847367789596, 0.00017526224837638438, 0.0001699505082797259, 0.00016466653323732316, 0.00016053428407758474, 0.00015698715287726372, 0.00015311270544771105, 0.00015072032692842185, 0.00014702034241054207, 0.00014439098595175892, 0.00014281776384450495, 0.00013998393842484802, 0.00013717943511437625, 0.00013519232743419707, 0.00013273413060232997, 0.00013038225006312132, 0.00012846071331296116, 0.00012696554767899215, 0.0001251263020094484, 0.00012399339175317436, 0.0001225267187692225, 0.00012142439663875848, 0.00011979944974882528, 0.00011861391249112785, 0.00011734178406186402, 0.00011631744564510882, 0.00011505914153531194, 0.00011418898066040128, 0.00011349399574100971, 0.00011282433843007311 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 0.00014675117381557357, "first_to_last_ratio": 1.2676837465168667, "imbalance_abs_log_ratio": 0.23719141364257146, "last_quartile_mean": 0.00011576323686313117, "mean": 0.00013042020850662084, "normalized": [ 1.169988069421413, 1.156175152925158, 1.1417665260225645, 1.1306726331060184, 1.1183621029407895, 1.1062694829504214, 1.0940115056230622, 1.0844989956600726, 1.0711591662817939, 1.0579765491085218, 1.0443900226505625, 1.035471661013805, 1.0260498632741444, 1.013992390086173, 1.0062479316494695, 0.9977001177294099, 0.9863163473759529, 0.9769959732660659, 0.9687585669786032, 0.9602628595878113, 0.9503845997255937, 0.9426104617668664, 0.9330243108845692, 0.9259760938667744, 0.914797625470175, 0.906289421438531, 0.8956357003805738, 0.888470885240542, 0.8806216558595509, 0.8756991509238615, 0.8712858841028995, 0.8681382926882502 ], "peak_layer": 1, "peak_normalized": 1.169988069421413, "population_cv": 0.09123006364857297, "rest_mean": 0.00013202972385678786, "spike_contrast": 0.9219804606102014, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012172882561571896, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00015259008796419948, 0.00015078860451467335, 0.00014890942838974297, 0.0001474625605624169, 0.00014585701865144074, 0.00014427989663090557, 0.00014268120867200196, 0.00014144058513920754, 0.00013970080181024969, 0.0001379815221298486, 0.00013620956451632082, 0.00013504642993211746, 0.0001338176371064037, 0.00013224509893916547, 0.00013123506505507976, 0.0001301202573813498, 0.00012863558367826045, 0.00012742001854348928, 0.00012634569429792464, 0.00012523768236860633, 0.00012394935765769333, 0.00012293545296415687, 0.00012168522516731173, 0.00012076599523425102, 0.00011930809705518186, 0.00011819845531135798, 0.00011680899478960782, 0.00011587455810513347, 0.00011485085997264832, 0.00011420886585256085, 0.00011363328667357564, 0.00011322277714498341 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00030002388848515693, "first_to_last_ratio": 2.2836575149168885, "imbalance_abs_log_ratio": 0.8257783308640597, "last_quartile_mean": 0.0001313786706305109, "mean": 0.0001947244400071213, "normalized": [ 1.8998833386658438, 1.7835645331795598, 1.6713073472446418, 1.5703972875779748, 1.4780250735507643, 1.3761669788219328, 1.3066447563403325, 1.2401013786965793, 1.1762106459027857, 1.150547637187856, 1.0768735950666006, 1.0264900078881174, 1.0140784159549234, 0.9666871251601796, 0.9096841069789114, 0.8754807117769676, 0.8421876125778792, 0.8073809902734768, 0.77911667677541, 0.7607651063686129, 0.7403822030704592, 0.7308855611321243, 0.7152791523187013, 0.7043381880055872, 0.6944157299089508, 0.686864629891504, 0.680599049276497, 0.6764772714226681, 0.6702412094778135, 0.6672258960044359, 0.6635863560708457, 0.6581114274310631 ], "peak_layer": 1, "peak_normalized": 1.8998833386658438, "population_cv": 0.3592853832280775, "rest_mean": 0.00020492727344390005, "spike_contrast": 0.6813594750078023, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013962913944851608, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0003699537192005664, 0.0003473036049399525, 0.0003254443872720003, 0.00030579473241232336, 0.00028780760476365685, 0.00026797334430739284, 0.0002544356684666127, 0.0002414780465187505, 0.0002290369593538344, 0.00022403974435292184, 0.00020969360775779933, 0.00019988269195891917, 0.0001974658516701311, 0.00018823760910890996, 0.00017713772831484675, 0.00017047749133780599, 0.00016399451124016196, 0.0001572168112033978, 0.00015171305858530104, 0.00014813955931458622, 0.00014417050988413393, 0.00014232128160074353, 0.00013928233238402754, 0.0001371518592350185, 0.00013521971413865685, 0.00013374933041632175, 0.00013252926873974502, 0.00013172665785532445, 0.00013051234418526292, 0.00012992518895771354, 0.00012921648158226162, 0.000128150379168801 ] } } }, "mlp_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "afe29f3ef4d2992353f0bb59fe7c5c2c5050fca671a53184825a8d63cd83a2bc", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.0027073738165199757, 0.002722575329244137, 0.0026850115973502398, 0.0026861224323511124, 0.00271838391199708, 0.0026645169127732515, 0.0026859452482312918, 0.0027359756641089916, 0.0026767742820084095, 0.0027451682835817337, 0.0026999011170119047, 0.002663193503394723, 0.002626648638397455, 0.0026531771291047335, 0.0026517712976783514, 0.002676903735846281, 0.002670086920261383, 0.0026355870068073273, 0.0026610493659973145, 0.0026388936676084995, 0.0026469926815479994, 0.00264862016774714, 0.002641494618728757, 0.002674198243767023, 0.002557571977376938, 0.00252733426168561, 0.0025468147359788418, 0.0026856842450797558, 0.002551089273765683, 0.0025778315030038357, 0.0026001501828432083, 0.0025662274565547705 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.00015858857113926206, "first_to_last_ratio": 3.757867690304466, "imbalance_abs_log_ratio": 1.3238516928952826, "last_quartile_mean": 4.220174423608114e-05, "mean": 8.573221236929385e-05, "normalized": [ 2.5461641896458365, 2.3032502265274117, 2.0924715218207024, 1.8886722264368014, 1.7031663195050915, 1.537434754996766, 1.4223955235407997, 1.3049501874408618, 1.2055297161045848, 1.1692542365736316, 1.0769571962730058, 1.0144505847411098, 0.9913601784485884, 0.9317001386730132, 0.857757082878185, 0.8155634736467603, 0.7682612945963089, 0.7217806418748633, 0.678338533281248, 0.6511820384364709, 0.6192882306865346, 0.6053793146733001, 0.5848721334223365, 0.5718144052517429, 0.5462281883927541, 0.527515083157495, 0.5112993585545178, 0.496920907994256, 0.47722358366421574, 0.46552107689700334, 0.4601234027970099, 0.45317424906679377 ], "peak_layer": 1, "peak_normalized": 2.5461641896458365, "population_cv": 0.5686091737198813, "rest_mean": 9.231269224848668e-05, "spike_contrast": 0.5437781067692293, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.0197621021652594e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00021828828903380781, 0.00019746273756027222, 0.00017939221288543195, 0.0001619200484128669, 0.00014601621660403907, 0.000131807682919316, 0.00012194511509733275, 0.00011187626660102978, 0.00010335272963857278, 0.00010024275252362713, 9.232992306351662e-05, 8.697109296917915e-05, 8.499150135321543e-05, 7.987671415321529e-05, 7.353741239057854e-05, 6.992006092332304e-05, 6.586474046343938e-05, 6.187985127326101e-05, 5.8155463193543255e-05, 5.582727681030519e-05, 5.309295011102222e-05, 5.1900507969548926e-05, 5.0142381951445714e-05, 4.902291402686387e-05, 4.682935104938224e-05, 4.522503513726406e-05, 4.3834825191879645e-05, 4.260212881490588e-05, 4.0913433622336015e-05, 3.991015182691626e-05, 3.944739728467539e-05, 3.885163096128963e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.00021242323236947414, "first_to_last_ratio": 1.8345651540958352, "imbalance_abs_log_ratio": 0.6068074801791611, "last_quartile_mean": 0.0001157894184871111, "mean": 0.00015111673860701558, "normalized": [ 1.771707915810251, 1.6366668487510905, 1.5242909365839221, 1.4186705636795862, 1.3264449439106374, 1.2453857912414694, 1.1893415085703798, 1.1330085309323608, 1.0858463851390703, 1.062861239127963, 1.023237806768965, 0.9942126959777651, 0.978436257513136, 0.9531027445378829, 0.9232414968561398, 0.904013237249231, 0.8853704564278231, 0.8680164030054869, 0.8515402788738432, 0.8395817783841374, 0.8267435189825344, 0.8185143642831039, 0.809172703595369, 0.8007918347797284, 0.7909896391756025, 0.7821052393452608, 0.7753388667989797, 0.7682453290925442, 0.7604995795745633, 0.7549901567959042, 0.7508572350862155, 0.7467737131490535 ], "peak_layer": 1, "peak_normalized": 1.771707915810251, "population_cv": 0.2714170004285523, "rest_mean": 0.00015645500981783562, "spike_contrast": 0.7816309251520459, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012229007406858728, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0002677347220014781, 0.00024732775636948645, 0.00023034587502479553, 0.00021438486874103546, 0.00020044803386554122, 0.0001881986390799284, 0.00017972940986510366, 0.00017121655400842428, 0.00016408956435043365, 0.00016061612404882908, 0.0001546283601783216, 0.00015024218009784818, 0.00014785809617023915, 0.0001440297783119604, 0.00013951724395155907, 0.000136611532070674, 0.0001337942958343774, 0.00013117180787958205, 0.0001286819897359237, 0.00012687486014328897, 0.00012493478425312787, 0.00012369122123345733, 0.00012227953993715346, 0.00012101305037504062, 0.00011953177454415709, 0.00011818919301731512, 0.00011716668086592108, 0.00011609472858253866, 0.00011492421617731452, 0.00011409165017539635, 0.00011346709652571008, 0.00011285000800853595 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.18837904930114746, "first_to_last_ratio": 1.8345651581276443, "imbalance_abs_log_ratio": 0.6068074823768531, "last_quartile_mean": 0.10268321540206671, "mean": 0.13401183625683188, "normalized": [ 1.7717078885231814, 1.6366668119471057, 1.524290957183596, 1.418670514273678, 1.326444920903592, 1.2453858718827995, 1.1893415500975306, 1.1330085716119305, 1.0858463365841577, 1.062861212873729, 1.0232377434953237, 0.9942127340740732, 0.9784362443707608, 0.9531027271661117, 0.9232414917376033, 0.904013241653558, 0.885370477646398, 0.8680164390463408, 0.8515402681847718, 0.8395816969955481, 0.8267435345782925, 0.8185144279720658, 0.8091727254708025, 0.8007918405918669, 0.7909896328038464, 0.7821052665924377, 0.7753388466443417, 0.7682453528580236, 0.7604996016741158, 0.7549901615676585, 0.7508572336421532, 0.7467736753526057 ], "peak_layer": 1, "peak_normalized": 1.7717078885231814, "population_cv": 0.27141699472708714, "rest_mean": 0.1387458680956452, "spike_contrast": 0.7816309473985974, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.10844806432724, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.23742982745170593, 0.2193327248096466, 0.2042730301618576, 0.19011864066123962, 0.1777593195438385, 0.16689644753932953, 0.15938584506511688, 0.151836559176445, 0.1455162614583969, 0.1424359828233719, 0.13712596893310547, 0.13323627412319183, 0.13112203776836395, 0.12772704660892487, 0.1237252876162529, 0.1211484745144844, 0.11865012347698212, 0.11632447689771652, 0.11411647498607635, 0.11251388490200043, 0.11079341918230057, 0.10969062149524689, 0.10843872278928757, 0.10731558501720428, 0.10600197315216064, 0.10481136292219162, 0.10390458256006241, 0.1029539704322815, 0.10191594809293747, 0.10117761790752411, 0.10062375664710999, 0.10007651150226593 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 6.873187476230669e-05, "first_to_last_ratio": 2.248757500145603, "imbalance_abs_log_ratio": 0.8103778415280068, "last_quartile_mean": 3.0564378221242805e-05, "mean": 4.685761763312257e-05, "normalized": [ 1.7400006587426948, 1.6405194725375354, 1.5551306973978087, 1.4944078341288562, 1.4327724019957897, 1.3414187773842097, 1.2804088050749736, 1.2499327782691996, 1.1927955171655855, 1.1608931065609767, 1.0981756972550076, 1.0577423904261343, 1.0542786038595535, 1.0218640087393278, 0.9775807389332813, 0.9570972402588515, 0.9209982861589985, 0.8815092249082567, 0.8478232134430647, 0.8204952153707635, 0.7871780668180614, 0.7759097745574741, 0.7506963524238334, 0.7421155343193898, 0.7072927325438048, 0.6891814936654451, 0.6695399899996352, 0.6616343983211914, 0.6345111343049281, 0.6212687927046295, 0.6200231913288072, 0.6148038704019305 ], "peak_layer": 1, "peak_normalized": 1.7400006587426948, "population_cv": 0.3217536161653951, "rest_mean": 4.900405669845727e-05, "spike_contrast": 0.719671983430414, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.526684668031521e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 8.153228554874659e-05, 7.687083416385576e-05, 7.286971958819777e-05, 7.002439087955281e-05, 6.71363013680093e-05, 6.285568815656006e-05, 5.999690620228648e-05, 5.8568872191244736e-05, 5.5891556257847697e-05, 5.439668530016206e-05, 5.145789691596292e-05, 4.9563288484932855e-05, 4.9400983698433265e-05, 4.7882112994557247e-05, 4.580710447044112e-05, 4.484729652176611e-05, 4.3155785533599555e-05, 4.130542220082134e-05, 3.972697595600039e-05, 3.844645107164979e-05, 3.688528886414133e-05, 3.635728353401646e-05, 3.517584264045581e-05, 3.477376594673842e-05, 3.314205241622403e-05, 3.229340290999971e-05, 3.1373048841487616e-05, 3.10026116494555e-05, 2.9731680115219206e-05, 2.9111175535945222e-05, 2.9052809622953646e-05, 2.8808244678657502e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 0.0001719075116852764, "first_to_last_ratio": 1.4894635014227988, "imbalance_abs_log_ratio": 0.39841598895878805, "last_quartile_mean": 0.00011541572621354135, "mean": 0.00013894635094402474, "normalized": [ 1.3738985779819257, 1.3296082759737424, 1.2883384004275333, 1.2499940655466777, 1.2130730176151365, 1.1755503979222366, 1.147409009170091, 1.11990595257071, 1.0929037170303153, 1.075598370036652, 1.0512762916941902, 1.0312594730951, 1.0194418974669357, 1.0018996501730997, 0.9803460168062212, 0.9648004557293595, 0.9493010807852901, 0.9344788929945884, 0.9198953859218769, 0.908001147597105, 0.8952913731815082, 0.886953676542665, 0.8771894463102521, 0.8683888934041172, 0.8579399877904575, 0.8482799643655919, 0.8408636339472488, 0.8330870307470335, 0.82434015220157, 0.8181187963300758, 0.8135579418994486, 0.809009026741244 ], "peak_layer": 1, "peak_normalized": 1.3738985779819257, "population_cv": 0.16161423472023242, "rest_mean": 0.0001421073115866252, "spike_contrast": 0.8576417505420824, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012187716347398237, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0001908981939777732, 0.0001847442181315273, 0.00017900991952046752, 0.00017368211410939693, 0.00016855206922627985, 0.00016333843814209104, 0.00015942829486448318, 0.00015560684551019222, 0.0001518549834145233, 0.00014945046859793365, 0.00014607100456487387, 0.0001432897406630218, 0.00014164773165248334, 0.00013921030040364712, 0.000136215501697734, 0.0001340555027127266, 0.0001319019211223349, 0.00012984243221580982, 0.0001278161071240902, 0.00012616344611160457, 0.00012439746933523566, 0.00012323897681199014, 0.00012188227265141904, 0.00012065946793882176, 0.00011920763063244522, 0.00011786540562752634, 0.00011683493357850239, 0.00011575440294109285, 0.0001145390560850501, 0.00011367462138878182, 0.00011304090730845928, 0.00011240885214647278 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 0.00014552221000485588, "first_to_last_ratio": 1.262697055715921, "imbalance_abs_log_ratio": 0.2332499537229619, "last_quartile_mean": 0.00011524712863320019, "mean": 0.00012953460168318998, "normalized": [ 1.1676213631634658, 1.1525055688027706, 1.1391580058081985, 1.1289307982409096, 1.1166731577821778, 1.1054001772932092, 1.094813144753513, 1.0822854389705017, 1.0679308603491235, 1.0564894822224282, 1.0456838360716472, 1.0337072702557388, 1.0234945545465333, 1.0158819477934218, 1.0051772952892828, 0.9951467950359763, 0.9866643397779161, 0.9787559418439278, 0.970448736962688, 0.9607908684400951, 0.9509939230158972, 0.9428061355388626, 0.934924980051271, 0.9261034265480621, 0.9170582603044295, 0.9058562786860701, 0.8981778407016658, 0.8907564927732574, 0.8831168120837269, 0.8778216107141017, 0.8745192645108896, 0.8703053916682415 ], "peak_layer": 1, "peak_normalized": 1.1676213631634658, "population_cv": 0.08967236486753827, "rest_mean": 0.00013110875065817877, "spike_contrast": 0.9231588022206524, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012103419721825049, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00015124736819416285, 0.00014928934979252517, 0.000147560378536582, 0.00014623560127802193, 0.00014464781270362437, 0.00014318757166620344, 0.00014181618462316692, 0.00014019341324456036, 0.0001383339986205101, 0.00013685194426216185, 0.00013545223919209093, 0.00013390085950959474, 0.00013257795944809914, 0.00013159186346456409, 0.00013020524056628346, 0.00012890594371128827, 0.00012780717224813998, 0.00012678276107180864, 0.0001257066905964166, 0.0001244556624442339, 0.00012318661902099848, 0.0001221260172314942, 0.00012110513489460573, 0.0001199624384753406, 0.00011879077646881342, 0.00011733973224181682, 0.00011634510883595794, 0.00011538378748809919, 0.0001143941844929941, 0.00011370827269274741, 0.00011328050459269434, 0.0001127346622524783 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00029741139769612346, "first_to_last_ratio": 2.278072415841521, "imbalance_abs_log_ratio": 0.8233296537390045, "last_quartile_mean": 0.00013055396993877366, "mean": 0.00019328556300024502, "normalized": [ 1.889398576482247, 1.780233894901915, 1.6772594716902607, 1.566224958959574, 1.4762232610465842, 1.375571144634086, 1.3083686481730985, 1.2364401527267181, 1.173148897934763, 1.1508864184295666, 1.080457040307468, 1.0318582947152708, 1.0081957004611959, 0.9698994715426978, 0.9066127339785177, 0.8754550048258108, 0.8446146862114313, 0.8115724449181009, 0.7806677558079268, 0.7613480996239201, 0.7403134026279838, 0.7294052762839086, 0.7160186976339193, 0.7062575707090604, 0.6942777320753531, 0.6866005528973866, 0.6829938476675228, 0.6753403086482458, 0.6715967314167978, 0.6686860556999406, 0.6640286432895989, 0.6600445236791291 ], "peak_layer": 1, "peak_normalized": 1.889398576482247, "population_cv": 0.3579154260008552, "rest_mean": 0.00020340604748559632, "spike_contrast": 0.681567477924495, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013863494677934796, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0003651934675872326, 0.0003440935106482357, 0.00032419004128314555, 0.0003027286729775369, 0.0002853326441254467, 0.00026587804313749075, 0.00025288877077400684, 0.00023898603103589267, 0.00022675274522043765, 0.00022244972933549434, 0.0002088367473334074, 0.00019944331143051386, 0.00019486967357806861, 0.00018746756541077048, 0.00017523515271022916, 0.00016921281348913908, 0.0001632518251426518, 0.00015686523693148047, 0.00015089180669747293, 0.00014715759607497603, 0.00014309189282357693, 0.00014098350948188454, 0.00013839607709087431, 0.0001365093921776861, 0.0001341938623227179, 0.00013270997442305088, 0.00013201285037212074, 0.00013053353177383542, 0.0001298099523410201, 0.00012924736074637622, 0.00012834715016651899, 0.00012757707736454904 ] } } }, "post_attention_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "31853ffb07b24cc59e774bb20656840b4358b3080b42516d380f2dc0f46404c8", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.0022265110164880753, 0.0040087271481752396, 0.005379301495850086, 0.006410556845366955, 0.0027253651060163975, 0.004838948138058186, 0.006175771821290255, 0.00742313964292407, 0.0030937574338167906, 0.005150026176124811, 0.007019025273621082, 0.008143599145114422, 0.004255065228790045, 0.0066224560141563416, 0.008129785768687725, 0.010099578648805618, 0.0041999719105660915, 0.00719039561226964, 0.009030314162373543, 0.010736005380749702, 0.005131533369421959, 0.00787449348717928, 0.009912566281855106, 0.012113703414797783, 0.0059690922498703, 0.00876442901790142, 0.010806876234710217, 0.012353328987956047, 0.006502739619463682, 0.009470963850617409, 0.012123330496251583, 0.01384514570236206 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.00015962236011546338, "first_to_last_ratio": 3.7698422272252903, "imbalance_abs_log_ratio": 1.3270331510418396, "last_quartile_mean": 4.2341920561739244e-05, "mean": 8.61727822893954e-05, "normalized": [ 2.542734798705971, 2.3221448837042376, 2.101145461274622, 1.8911858524935854, 1.7004204056817789, 1.5354142426015858, 1.4229001770142031, 1.3028733810328659, 1.2049412416934622, 1.1705258152816296, 1.0773908232649334, 1.0133220774376726, 0.990609360100461, 0.9272534949358001, 0.8550385909364457, 0.8185651575045715, 0.7703972051466339, 0.7173947009730917, 0.6760256695667757, 0.6514523109028573, 0.618419077603279, 0.6053620388084041, 0.5838930841859182, 0.5697043695032278, 0.5457802700901755, 0.5276149005446632, 0.5104701332743111, 0.49517988582787786, 0.47527562953524777, 0.46579059316028215, 0.45896435764104826, 0.45181000957238127 ], "peak_layer": 1, "peak_normalized": 2.542734798705971, "population_cv": 0.5705202801130907, "rest_mean": 9.280119640472414e-05, "spike_contrast": 0.542873885449774, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.03793460666202e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00021911453222855926, 0.00020010568550787866, 0.00018106155039276928, 0.00016296874673571438, 0.00014652995741926134, 0.00013231091725174338, 0.0001226152671733871, 0.00011227222421439365, 0.00010383313929196447, 0.00010086746624438092, 9.284176485380158e-05, 8.732078276807442e-05, 8.536356472177431e-05, 7.99040135461837e-05, 7.368105434579775e-05, 7.053803710732609e-05, 6.638727063545957e-05, 6.181989738252014e-05, 5.825501284562051e-05, 5.613745815935545e-05, 5.329089253791608e-05, 5.216573117650114e-05, 5.0315691623836756e-05, 4.9093010602518916e-05, 4.703140439232811e-05, 4.546604395727627e-05, 4.398863165988587e-05, 4.267102849553339e-05, 4.095582335139625e-05, 4.013847137684934e-05, 3.955023566959426e-05, 3.893372559105046e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.00021442477373057045, "first_to_last_ratio": 1.8431254562763404, "imbalance_abs_log_ratio": 0.6114627481432037, "last_quartile_mean": 0.00011633759004325839, "mean": 0.00015221248349917005, "normalized": [ 1.7740805953138137, 1.6484457246979003, 1.5301434809496548, 1.4217241600875643, 1.3264958046529274, 1.2454577490392524, 1.1897861330408546, 1.1336265611690741, 1.086914920172498, 1.064301358569379, 1.0227209980571423, 0.9941753882589759, 0.9788979908211966, 0.9511107967643203, 0.9228143274453928, 0.9052447611165039, 0.8846530997418666, 0.8652930871753776, 0.8497381552743479, 0.8388656494639406, 0.8256334792987466, 0.8176309647342147, 0.8076951735882674, 0.8000661307359944, 0.7892605467734257, 0.78144613040264, 0.7731219607566042, 0.7664988488799842, 0.7584935618388989, 0.7529639051546407, 0.7485195296370439, 0.7441790263875565 ], "peak_layer": 1, "peak_normalized": 1.7740805953138137, "population_cv": 0.2736411578809574, "rest_mean": 0.00015762286892384981, "spike_contrast": 0.7803207938393821, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012299640220589936, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00027003721334040165, 0.00025091401766985655, 0.00023290693934541196, 0.0002164041652576998, 0.000201909220777452, 0.0001895742170745507, 0.00018110030214302242, 0.00017255211423616856, 0.00016544201935175806, 0.00016199995297938585, 0.00015567090304102749, 0.00015132590488065034, 0.0001490004942752421, 0.00014477093645837158, 0.00014046386058907956, 0.00013778955326415598, 0.0001346552453469485, 0.00013170840975362808, 0.00012934075493831187, 0.00012768582382705063, 0.00012567172234412283, 0.00012445363972801715, 0.00012294128828216344, 0.00012178005272289738, 0.00012013530795229599, 0.00011894585622940212, 0.00011767881369451061, 0.00011667069338727742, 0.00011545218876563013, 0.00011461050598882139, 0.00011393401655368507, 0.00011327333777444437 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.190154030919075, "first_to_last_ratio": 1.84312545379361, "imbalance_abs_log_ratio": 0.6114627467961817, "last_quartile_mean": 0.10316933691501617, "mean": 0.13498355145566165, "normalized": [ 1.774080628234443, 1.6484456620184311, 1.5301434020504485, 1.4217242444591272, 1.326495766996536, 1.2454576834375484, 1.189786122599316, 1.133626627255238, 1.0869149569673349, 1.0643014004698326, 1.0227209907477024, 0.9941753870542451, 0.9788979589698585, 0.9511107512164827, 0.9228143032197887, 0.9052447972745712, 0.8846531331847195, 0.8652931173486506, 0.8497382179112226, 0.8388656682549475, 0.8256334805754935, 0.8176309131099365, 0.8076951535479161, 0.8000661540388709, 0.7892605573866218, 0.7814461538045975, 0.7731219580171443, 0.7664988542586116, 0.7584935821787124, 0.7529638601652359, 0.7485195164462104, 0.7441789968002049 ], "peak_layer": 1, "peak_normalized": 1.774080628234443, "population_cv": 0.27364115345546325, "rest_mean": 0.13978153549962574, "spike_contrast": 0.7803207857775153, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.10907443761825561, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.2394717037677765, 0.22251304984092712, 0.2065441906452179, 0.19190938770771027, 0.1790551096200943, 0.16811630129814148, 0.16060155630111694, 0.1530209481716156, 0.14671564102172852, 0.1436631828546524, 0.13805051147937775, 0.13419732451438904, 0.13213512301445007, 0.1283843070268631, 0.1245647519826889, 0.12219315767288208, 0.11941362172365189, 0.11680033802986145, 0.11470068246126175, 0.1132330670952797, 0.11144693940877914, 0.11036672443151474, 0.10902556031942368, 0.10799577087163925, 0.10653719305992126, 0.10548237711191177, 0.1043587476015091, 0.10346473753452301, 0.10238415747880936, 0.10163773596286774, 0.10103782266378403, 0.10045192390680313 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 6.883367768750759e-05, "first_to_last_ratio": 2.249759357551583, "imbalance_abs_log_ratio": 0.810823258297238, "last_quartile_mean": 3.059601795030176e-05, "mean": 4.691846504556452e-05, "normalized": [ 1.7359059710084281, 1.6475789727938472, 1.5536456441204052, 1.4921194549590373, 1.4365657321885223, 1.3438513058553838, 1.2787343947807137, 1.2483299149152907, 1.1912354282513808, 1.1603083401306369, 1.099547144002948, 1.059972440022979, 1.0583955432739494, 1.0176800955770247, 0.9754963047746087, 0.9631595708780456, 0.9242400656751482, 0.8752301116137658, 0.8454245377074081, 0.8232853322610043, 0.7863494898171388, 0.7774642190168813, 0.7497486129019063, 0.7388483538626245, 0.709315327139865, 0.6908343794807076, 0.6685469983108738, 0.6592365073812129, 0.6314341746588822, 0.6235898169926835, 0.6199474545603838, 0.6139783610863121 ], "peak_layer": 1, "peak_normalized": 1.7359059710084281, "population_cv": 0.3220406003070692, "rest_mean": 4.9070239684725595e-05, "spike_contrast": 0.7193541792517962, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.52988819940947e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 8.144604362314567e-05, 7.730187644483522e-05, 7.28946688468568e-05, 7.000795449130237e-05, 6.740145909134299e-05, 6.305144052021205e-05, 5.999625500408001e-05, 5.8569723478285596e-05, 5.589093780145049e-05, 5.4439886298496276e-05, 5.158906424185261e-05, 4.973227987647988e-05, 4.9658294301480055e-05, 4.774798799189739e-05, 4.576878927764483e-05, 4.51899686595425e-05, 4.3363925215089694e-05, 4.1064453398576006e-05, 3.9666021621087566e-05, 3.86272840842139e-05, 3.689431105158292e-05, 3.6477427784120664e-05, 3.5177054087398574e-05, 3.466563066467643e-05, 3.3279986382694915e-05, 3.2412888685939834e-05, 3.136719897156581e-05, 3.093036502832547e-05, 2.9625922252307646e-05, 2.9257877031341195e-05, 2.9086982976878062e-05, 2.8806922273361124e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 0.00017341253987979144, "first_to_last_ratio": 1.4955148931299893, "imbalance_abs_log_ratio": 0.4024705576674927, "last_quartile_mean": 0.00011595507385209203, "mean": 0.00013990553861731314, "normalized": [ 1.3768818258178377, 1.3340879911950096, 1.2910054180628048, 1.2527236427111486, 1.2147542166907819, 1.1769889670731195, 1.1474411155803916, 1.1220953663134672, 1.0944075691302455, 1.0773007485615413, 1.0508678613623372, 1.0320596054419082, 1.0208161736522385, 1.000559648637946, 0.9805197815181176, 0.9663128275349431, 0.948744191771126, 0.9319305822463463, 0.9182019814348666, 0.9075038868226692, 0.8943650364968407, 0.886262676120959, 0.8757907045344768, 0.8679002174812178, 0.856288373659771, 0.8478178092769747, 0.8387215581135022, 0.8313975747674917, 0.8224084044020752, 0.8161864847722404, 0.8112200992010112, 0.8064376596145924 ], "peak_layer": 1, "peak_normalized": 1.3768818258178377, "population_cv": 0.16325888721442564, "rest_mean": 0.0001431150389921472, "spike_contrast": 0.8564734877369171, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001225742365932092, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0001926333934534341, 0.00018664629897102714, 0.00018061880837194622, 0.0001752629759721458, 0.00016995084297377616, 0.0001646672753849998, 0.00016053336730692536, 0.00015698735660407692, 0.00015311368042603135, 0.00015072034148033708, 0.00014702223415952176, 0.00014439085498452187, 0.0001428178366040811, 0.00013998383656144142, 0.00013718014815822244, 0.00013519251660909504, 0.00013273456715978682, 0.00013038225006312132, 0.00012846154277212918, 0.00012696482008323073, 0.00012512662215158343, 0.0001239930570591241, 0.00012252797023393214, 0.000121424047392793, 0.00011979948612861335, 0.00011861440725624561, 0.00011734179133782163, 0.0001163171255029738, 0.00011505949078127742, 0.00011418900976423174, 0.00011349418491590768, 0.00011282509512966499 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 0.00014675183047074825, "first_to_last_ratio": 1.2676545313577883, "imbalance_abs_log_ratio": 0.23716836728277432, "last_quartile_mean": 0.00011576642282307148, "mean": 0.00013041823922321782, "normalized": [ 1.1700698937843759, 1.1560440994564782, 1.1417398043557874, 1.130463200949706, 1.118403537277814, 1.1065304334184665, 1.0939881912841023, 1.0846815122144253, 1.0709468270677047, 1.0579907390333845, 1.0442838370166418, 1.035550673160169, 1.025959021745254, 1.0140203095307772, 1.0062047700154308, 0.9976631870190885, 0.9861765922450596, 0.9770016877798172, 0.9687702939454803, 0.9602422119882037, 0.9504995944091904, 0.9425883202531964, 0.9330014667281973, 0.9259385264566372, 0.9147443798098084, 0.906202796813466, 0.8958065504175365, 0.8885655303249116, 0.8806194435723184, 0.8757905905296257, 0.8713783728577705, 0.8681336045391749 ], "peak_layer": 1, "peak_normalized": 1.1700698937843759, "population_cv": 0.09121429133712369, "rest_mean": 0.0001320278307604086, "spike_contrast": 0.921975648780332, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001217264449223876, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0001525984553154558, 0.0001507692359155044, 0.000148903694935143, 0.00014743302017450333, 0.00014586022007279098, 0.00014431175077334046, 0.00014267601363826543, 0.00014146225294098258, 0.000139670999487862, 0.00013798128929920495, 0.00013619365927297622, 0.00013505469541996717, 0.00013380376913119107, 0.00013224674330558628, 0.00013122745440341532, 0.00013011347618885338, 0.00012861541472375393, 0.00012741883983835578, 0.0001263453159481287, 0.0001252330985153094, 0.0001239624834852293, 0.00012293070903979242, 0.00012168040848337114, 0.00012075927224941552, 0.00011929935135412961, 0.00011818537313956767, 0.00011682951299007982, 0.00011588515189941972, 0.00011484883725643158, 0.00011421906674513593, 0.00011364363308530301, 0.00011322045611450449 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00030004180916876066, "first_to_last_ratio": 2.2837506327398005, "imbalance_abs_log_ratio": 0.8258191057718037, "last_quartile_mean": 0.00013138116082700435, "mean": 0.00019473525253488333, "normalized": [ 1.9000096511818072, 1.783700741635724, 1.6708066910504589, 1.570366884717585, 1.4783058799917985, 1.376185321596543, 1.3064973297630273, 1.2402700043336092, 1.1767953849206394, 1.1505341943958838, 1.0771542571420398, 1.026098162637436, 1.013898063759585, 0.9669473025647412, 0.9095398902231254, 0.8755205777387188, 0.842301214161826, 0.8073124727024382, 0.7791991818813049, 0.760574607765185, 0.7402055397447304, 0.7308143413325056, 0.7152667122155245, 0.7043714155678553, 0.6942467749765324, 0.6868749151283189, 0.680655564574882, 0.676363713518914, 0.6701061029149729, 0.6672276319278244, 0.6636177363814285, 0.6582317375530352 ], "peak_layer": 1, "peak_normalized": 1.9000096511818072, "population_cv": 0.35930746026536914, "rest_mean": 0.00020494150898971216, "spike_contrast": 0.6812747128050888, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013962146767880766, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0003699988592416048, 0.0003473494143690914, 0.0003253649629186839, 0.0003058057918678969, 0.0002878782688640058, 0.0002679917961359024, 0.0002544210874475539, 0.00024152429250534624, 0.00022916354646440595, 0.000224049566895701, 0.00020975990628357977, 0.00019981748482678086, 0.000197441695490852, 0.0001882987271528691, 0.00017711948021315038, 0.00017049472080543637, 0.00016402573965024203, 0.00015721219824627042, 0.00015173754945863038, 0.00014811068831477314, 0.00014414411270990968, 0.0001423153153154999, 0.0001392876438330859, 0.00013716594548895955, 0.00013519432104658335, 0.00013375876005738974, 0.00013254763325676322, 0.0001317118585575372, 0.00013049328117631376, 0.00012993274140171707, 0.00012922976748086512, 0.00012818092363886535 ] } } }, "post_mlp_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "7fd9c911e93175896ad618842c120e77b3f017c17b7eec9c2bfb01fc6dbc8ac2", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.0034885562490671873, 0.004831995815038681, 0.005952076520770788, 0.006925336085259914, 0.00385080324485898, 0.005542940925806761, 0.006729288958013058, 0.007927131839096546, 0.004081308841705322, 0.005811207927763462, 0.007534808944910765, 0.008581922389566898, 0.00499920267611742, 0.007102519739419222, 0.008542952127754688, 0.010447698645293713, 0.005041283555328846, 0.007636935915797949, 0.009387043304741383, 0.011020516976714134, 0.005807018838822842, 0.008367395959794521, 0.01024632342159748, 0.012378672137856483, 0.006453148555010557, 0.009127611294388771, 0.011037957854568958, 0.012611051090061665, 0.007010402157902718, 0.009794782847166061, 0.012445803731679916, 0.014087481424212456 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.00015858902861509705, "first_to_last_ratio": 3.757869257587208, "imbalance_abs_log_ratio": 1.3238521099622325, "last_quartile_mean": 4.220184837322449e-05, "mean": 8.57325238712292e-05, "normalized": [ 2.5462257183789148, 2.3032637538343064, 2.0924289533379263, 1.8886492391629932, 1.7031256747388024, 1.5374176267930122, 1.4224231993460965, 1.3049597038569594, 1.2055618292025094, 1.169245405308266, 1.0769812048488787, 1.0144334047898542, 0.9913512297333369, 0.9317090592952362, 0.8577499774884012, 0.81555804918997, 0.7682725064221458, 0.7217807351250844, 0.6783324192661732, 0.6511800118936032, 0.6192916242842709, 0.6053728292385717, 0.5848720876072855, 0.5718124973482859, 0.5462302773879167, 0.5275173250132701, 0.5112998770983732, 0.49691833866078555, 0.47721658788927634, 0.46551989467389665, 0.46012635628943105, 0.45317260249616614 ], "peak_layer": 1, "peak_normalized": 2.5462257183789148, "population_cv": 0.5686111305901927, "rest_mean": 9.231303704803361e-05, "spike_contrast": 0.5437775023084314, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.019775271648541e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00021829435718245804, 0.00019746461475733668, 0.0001793892151908949, 0.0001619186659809202, 0.00014601326256524771, 0.00013180669338908046, 0.00012194793089292943, 0.00011187748896190897, 0.00010335585830034688, 0.00010024235962191597, 9.233231685357168e-05, 8.69699360919185e-05, 8.499104296788573e-05, 7.987776916706935e-05, 7.353707042057067e-05, 6.991984992055222e-05, 6.586594099644572e-05, 6.188008410390466e-05, 5.8155150327365845e-05, 5.582730591413565e-05, 5.309343396220356e-05, 5.190014053368941e-05, 5.014256021240726e-05, 4.90229285787791e-05, 4.682970029534772e-05, 4.522539165918715e-05, 4.383502891869284e-05, 4.2602063331287354e-05, 4.0912982512963936e-05, 3.991019548266195e-05, 3.944779382436536e-05, 3.885163096128963e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.00021242368711682502, "first_to_last_ratio": 1.8345658968465297, "imbalance_abs_log_ratio": 0.6068078850437921, "last_quartile_mean": 0.0001157896194854402, "mean": 0.0001511171253696375, "normalized": [ 1.7717451736709768, 1.636674793183293, 1.5242755762025908, 1.4186515254985965, 1.3264266232459345, 1.2453738409548603, 1.1893512719369177, 1.1330135273991528, 1.085860457802012, 1.0628542818802846, 1.023252039669589, 0.9942070699734702, 0.9784308644775869, 0.9531083940389613, 0.9232390376562555, 0.9040072643238835, 0.8853775311228679, 0.8680181295617843, 0.8515367513400655, 0.8395795332993111, 0.8267439067367663, 0.8185133286645813, 0.8091730400239778, 0.800791325997512, 0.7909899739933552, 0.7821072339266707, 0.7753403490721149, 0.7682422073314642, 0.7604971517068638, 0.7549863467424895, 0.7508596466770359, 0.7467718018887739 ], "peak_layer": 1, "peak_normalized": 1.7717451736709768, "population_cv": 0.2714186609524326, "rest_mean": 0.00015645541295978345, "spike_contrast": 0.7816308178118705, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012229037238284946, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0002677410375326872, 0.0002473295899108052, 0.00023034414334688336, 0.00021438254043459892, 0.00020044577831868082, 0.00018819731485564262, 0.00017973134526982903, 0.000171217747265473, 0.0001640921109355986, 0.00016061548376455903, 0.00015463090676348656, 0.00015024171443656087, 0.0001478576596127823, 0.00014403100067283958, 0.00013951722939964384, 0.00013661097909789532, 0.00013379570737015456, 0.0001311724045081064, 0.0001286817860091105, 0.00012687484559137374, 0.0001249351626029238, 0.00012369138130452484, 0.00012227990373503417, 0.00012101328320568427, 0.00011953213106608018, 0.0001181897969217971, 0.0001171672047348693, 0.00011609455395955592, 0.00011492414341773838, 0.0001140913664130494, 0.00011346775136189535, 0.00011285000800853595 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.18837944976985455, "first_to_last_ratio": 1.8345658800697515, "imbalance_abs_log_ratio": 0.6068078758989703, "last_quartile_mean": 0.10268339328467846, "mean": 0.1340121782850474, "normalized": [ 1.7717451751532867, 1.6366747548098712, 1.5242756140248972, 1.4186514377480737, 1.3264266357207033, 1.2453739091726768, 1.1893513017871604, 1.1330134634090196, 1.0858604665419038, 1.0628542749024097, 1.0232520332438115, 0.9942070832344971, 0.9784308561813967, 0.9531084116998004, 0.9232390798282929, 0.9040072094634526, 0.8853775581660035, 0.8680181154203164, 0.8515367049614931, 0.8395794986042229, 0.8267439263823189, 0.8185133396785964, 0.809173050926764, 0.8007913534932437, 0.7909900046670136, 0.782107217826699, 0.7753403703776178, 0.7682422246066585, 0.7604971603471317, 0.7549863443945886, 0.7508596538022533, 0.7467717694238248 ], "peak_layer": 1, "peak_normalized": 1.7717451751532867, "population_cv": 0.27141865758333467, "rest_mean": 0.13874622461972413, "spike_contrast": 0.7816308398662988, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.10844832807779312, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.23743543028831482, 0.21933434903621674, 0.20427149534225464, 0.19011656939983368, 0.17775732278823853, 0.16689527034759521, 0.15938755869865417, 0.15183760225772858, 0.14551852643489838, 0.14243541657924652, 0.13712823390960693, 0.1332358568906784, 0.1311216503381729, 0.12772813439369202, 0.1237252801656723, 0.12114797532558441, 0.1186513751745224, 0.1163249984383583, 0.11411628872156143, 0.11251387745141983, 0.11079375445842743, 0.10969075560569763, 0.10843904316425323, 0.107315793633461, 0.10600229352712631, 0.104811891913414, 0.10390505194664001, 0.10295381397008896, 0.1019158810377121, 0.10117736458778381, 0.10062433779239655, 0.10007651150226593 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 6.873207303215167e-05, "first_to_last_ratio": 2.248757814140914, "imbalance_abs_log_ratio": 0.8103779811585758, "last_quartile_mean": 3.0564462122129044e-05, "mean": 4.6857780716891284e-05, "normalized": [ 1.7400119939228862, 1.6405409364325143, 1.555121247708426, 1.4944054279879166, 1.4327475398601994, 1.3414100714975594, 1.2804101716466747, 1.2499370459040815, 1.192804641985012, 1.1608858830017554, 1.098191983597808, 1.057744066133373, 1.0542717513613309, 1.021869458338179, 0.9775749297615711, 0.9570927445976433, 0.9210091333285576, 0.8815125232807631, 0.847820728512489, 0.8204970180441978, 0.7871838673851995, 0.7759041238100245, 0.7506938949785397, 0.7421070509174572, 0.7072987335050381, 0.6891811912847667, 0.6695407652827761, 0.6616333377893114, 0.6345027924930273, 0.6212629426015038, 0.6200302724102036, 0.614801730639213 ], "peak_layer": 1, "peak_normalized": 1.7400119939228862, "population_cv": 0.32175451551289735, "rest_mean": 4.9004235565748616e-05, "spike_contrast": 0.7196709452950106, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.5266924533061685e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 8.153310045599937e-05, 7.687210745643824e-05, 7.28695304132998e-05, 7.002452184678987e-05, 6.713537004543468e-05, 6.28554989816621e-05, 5.9997179050697014e-05, 5.8569276006892323e-05, 5.589217835222371e-05, 5.439653614303097e-05, 5.1458839152473956e-05, 4.9563539505470544e-05, 4.9400834541302174e-05, 4.788253500009887e-05, 4.58069916930981e-05, 4.4847241952084005e-05, 4.315644400776364e-05, 4.130572051508352e-05, 3.972699778387323e-05, 3.8446669350378215e-05, 3.688568904181011e-05, 3.635714529082179e-05, 3.5175849916413426e-05, 3.477348946034908e-05, 3.3142448955914006e-05, 3.2293501135427505e-05, 3.13731943606399e-05, 3.1002669857116416e-05, 2.9731392714893445e-05, 2.9111002731951885e-05, 2.905324254243169e-05, 2.8808244678657502e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 0.00017190765356644988, "first_to_last_ratio": 1.4894621368061003, "imbalance_abs_log_ratio": 0.3984150727783497, "last_quartile_mean": 0.00011541592721187044, "mean": 0.00013894661265112518, "normalized": [ 1.3739040544627474, 1.3296102750467884, 1.2883342981472405, 1.2499918159027226, 1.2130627732820631, 1.175544413473833, 1.1474083142393565, 1.1199112790671786, 1.092909618037032, 1.0755929927697745, 1.0512861461262244, 1.0312571117882559, 1.0194388253067554, 1.0019061415077835, 0.980344379776871, 0.9647961249933756, 0.9493075664595542, 0.9344817410273759, 0.9198929201794295, 0.9079992279088673, 0.8952920956888009, 0.8869530532606651, 0.8771902552778059, 0.8683889858357474, 0.8579407806493952, 0.8482823463732029, 0.8408658204630546, 0.8330843095877946, 0.8243382853592501, 0.818115422616924, 0.8135611224181157, 0.8090075029660142 ], "peak_layer": 1, "peak_normalized": 1.3739040544627474, "population_cv": 0.16161403917150835, "rest_mean": 0.00014210757001786045, "spike_contrast": 0.8576421569620596, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012187744287075474, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00019089931447524577, 0.00018474484386388212, 0.00017900968668982387, 0.00017368212866131216, 0.0001685509632807225, 0.00016333791427314281, 0.00015942849859129637, 0.00015560787869617343, 0.00015185608936008066, 0.00014945000293664634, 0.00014607264893129468, 0.0001432896824553609, 0.00014164757158141583, 0.0001392114645568654, 0.00013621553080156446, 0.0001340551534667611, 0.00013190307072363794, 0.00012984307250007987, 0.0001278160052606836, 0.00012616341700777411, 0.0001243978040292859, 0.00012323912233114243, 0.00012188261462142691, 0.00012065970804542303, 0.00011920796532649547, 0.00011786595860030502, 0.00011683545744745061, 0.00011575424287002534, 0.00011453901242930442, 0.00011367436673026532, 0.00011304156214464456, 0.00011240885214647278 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 0.00014552358334185556, "first_to_last_ratio": 1.262638843036328, "imbalance_abs_log_ratio": 0.2332038508028458, "last_quartile_mean": 0.00011525352965691127, "mean": 0.00012953498708156985, "normalized": [ 1.1676304712442283, 1.1525836984023852, 1.1391521450612143, 1.1288261090295841, 1.1167600441451297, 1.1054423860149858, 1.094731137320412, 1.082319740349084, 1.067957453000188, 1.0563935463552665, 1.04560972624621, 1.0337959762176818, 1.0235535208863231, 1.0159040893770892, 1.0050628637064216, 0.9950792389307415, 0.9865814183778941, 0.9786230528286792, 0.9704418054120744, 0.9608381133357455, 0.9508407831226554, 0.9429749854457838, 0.9346839821902475, 0.9262276149685831, 0.9170542960948607, 0.9059488475652709, 0.8984648923618161, 0.8908037775288885, 0.8831483920181569, 0.8778579808382188, 0.8744051093280881, 0.8703028022960919 ], "peak_layer": 1, "peak_normalized": 1.1676304712442283, "population_cv": 0.0896642712448568, "rest_mean": 0.00013110964479252557, "spike_contrast": 0.9231344927669946, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012103183544240892, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00015124899800866842, 0.00014929991448298097, 0.00014756005839444697, 0.00014622247545048594, 0.00014465949789155275, 0.00014319346519187093, 0.00014180598373059183, 0.00014019827358424664, 0.00013833785487804562, 0.00013683992438018322, 0.0001354430423816666, 0.00013391274842433631, 0.0001325859921053052, 0.00013159512309357524, 0.00013019080506637692, 0.00012889757636003196, 0.00012779681128449738, 0.00012676592450588942, 0.00012570616672746837, 0.00012446215259842575, 0.00012316714855842292, 0.00012214825255796313, 0.00012107427755836397, 0.00011997888213954866, 0.00011879061639774591, 0.00011735207226593047, 0.0001163826382253319, 0.0001153902558144182, 0.00011439861555118114, 0.00011371332220733166, 0.00011326605454087257, 0.0001127346622524783 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00029742828155576717, "first_to_last_ratio": 2.2781890121807993, "imbalance_abs_log_ratio": 0.8233808344454435, "last_quartile_mean": 0.00013055469935352448, "mean": 0.00019329176166138495, "normalized": [ 1.8891850069388627, 1.7801603926271052, 1.6773544463608039, 1.566500262900107, 1.476456430986117, 1.3754925511314788, 1.3084504582544147, 1.236424592515421, 1.172965600373877, 1.150871569148191, 1.0802294364746856, 1.0319209662753672, 1.0080063247818516, 0.9698985570693867, 0.9067931019504507, 0.8756163462587222, 0.8444435806568995, 0.8113939671049883, 0.7809677247391495, 0.7613442367276676, 0.7402582678385915, 0.7293562882295931, 0.7162426695419845, 0.7062419232587185, 0.6940340549797547, 0.6866100033591334, 0.6831688143117768, 0.6753320518823956, 0.6715919825156708, 0.6686410475337363, 0.6640239865195754, 0.6600233567535221 ], "peak_layer": 1, "peak_normalized": 1.8891850069388627, "population_cv": 0.357915029863326, "rest_mean": 0.00020341356920888993, "spike_contrast": 0.6815376252628051, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001386340009048581, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00036516389809548855, 0.0003440903383307159, 0.0003242187958676368, 0.0003027915954589844, 0.00028538686456158757, 0.00026587137836031616, 0.0002529126941226423, 0.00023899068764876574, 0.00022672458726447076, 0.00022245399304665625, 0.0002087994507746771, 0.00019946182146668434, 0.00019483931828290224, 0.00018747340072877705, 0.00017527563613839447, 0.00016924942610785365, 0.00016322398732881993, 0.00015683576930314302, 0.0001509546273155138, 0.00014716156874783337, 0.0001430858246749267, 0.0001409785618307069, 0.00013844380737282336, 0.00013651074550580233, 0.00013415106514003128, 0.00013271605712361634, 0.00013205090363044292, 0.00013053612201474607, 0.00012981319741811603, 0.00012924280599690974, 0.00012835036613978446, 0.00012757707736454904 ] } } }, "pre_attention_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "de71d4cf2d509a1063f6eac7f50c4885e9a058dee7e4b993ad40f5527f528d00", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.02802751213312149, 0.01413465291261673, 0.014225807972252369, 0.01433455292135477, 0.01440478302538395, 0.009703479707241058, 0.00977881159633398, 0.009858808480203152, 0.009950309991836548, 0.00752287870272994, 0.007571155205368996, 0.007693085819482803, 0.007788818795233965, 0.006353048607707024, 0.006389216054230928, 0.0064707002602517605, 0.00656962301582098, 0.005553483963012695, 0.0056249531917274, 0.005688921548426151, 0.005768724251538515, 0.004966245498508215, 0.005057469941675663, 0.005098348017781973, 0.005216081161051989, 0.0046758754178881645, 0.004727107938379049, 0.004777040798217058, 0.004837613552808762, 0.0044186655431985855, 0.004510571248829365, 0.0045849340967834 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.0001580166108396952, "first_to_last_ratio": 1.9713740353643698, "imbalance_abs_log_ratio": 0.6787307795042399, "last_quartile_mean": 8.015557068574708e-05, "mean": 0.00011843875881822896, "normalized": [ 0.8918679597819287, 1.5814819593588525, 1.5722239987443252, 1.3246067564285042, 1.1252650527356378, 1.512743589128775, 1.3788112428745087, 1.2863035124436846, 1.097095055843533, 1.468154353173571, 1.1523254606068964, 1.0839888571909224, 1.054518217028451, 1.146335080549274, 1.1665208527784754, 0.8931136212023528, 0.8203804970547371, 1.073741223277434, 0.919654742394771, 0.8223971330190447, 0.7711288002244273, 0.8470775332584999, 0.8298711629738247, 0.7662487463744663, 0.7210382462310316, 0.7071376059209309, 0.7189559392079661, 0.727980518763353, 0.6111916361855597, 0.6986829946379066, 0.6244678178312321, 0.6046898327751223 ], "peak_layer": 2, "peak_normalized": 1.5814819593588525, "population_cv": 0.2904715201464359, "rest_mean": 0.00012310891098523926, "spike_contrast": 0.7572151875143335, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.321993711637333e-05, "top_five_layers": [ 2, 3, 6, 10, 7 ], "top_five_spike_overlap": 0 }, "values": [ 0.00010563173418631777, 0.0001873087603598833, 0.00018621225899551064, 0.00015688478015363216, 0.0001332749961875379, 0.00017916747310664505, 0.00016330469225067645, 0.00015234819147735834, 0.00012993857671972364, 0.00017388637934345752, 0.0001364799973089248, 0.0001283862948184833, 0.00012489582877606153, 0.00013577050413005054, 0.00013816128193866462, 0.00010577926877886057, 9.71648478298448e-05, 0.00012717257777694613, 0.00010892276623053476, 9.74036956904456e-05, 9.133153798757121e-05, 0.0001003268116619438, 9.828891052166e-05, 9.07535504666157e-05, 8.539887494407594e-05, 8.375250035896897e-05, 8.515224908478558e-05, 8.622110908618197e-05, 7.238877878990024e-05, 8.275114669231698e-05, 7.396119326585904e-05, 7.161871326388791e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.0001664285200604354, "first_to_last_ratio": 1.9719337364651075, "imbalance_abs_log_ratio": 0.6790146534176558, "last_quartile_mean": 8.439863722742302e-05, "mean": 0.0001245867656507471, "normalized": [ 0.8984334461398115, 1.5927952483854981, 1.5658595485525297, 1.3198735865176374, 1.131213894321221, 1.5125315092450449, 1.3796810696180641, 1.286366052942558, 1.0960606282761096, 1.4536884557404346, 1.1543342746982923, 1.0797559630746076, 1.0497530570909623, 1.1457182987306291, 1.1551962694199982, 0.9023721663305979, 0.8238710779144349, 1.0632503993742801, 0.9211624832492573, 0.8274335806257342, 0.7739032412019898, 0.850990856837025, 0.8281317613098801, 0.7681943949939335, 0.7208251570035091, 0.7089546837555952, 0.7203162530736348, 0.7236174125342815, 0.6145740290254166, 0.6994991973458914, 0.6274381906330746, 0.6042038120380651 ], "peak_layer": 2, "peak_normalized": 1.5927952483854981, "population_cv": 0.2889665774838781, "rest_mean": 0.00012946851159169995, "spike_contrast": 0.7586812913967177, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.822533756960183e-05, "top_five_layers": [ 2, 3, 6, 10, 7 ], "top_five_spike_overlap": 0 }, "values": [ 0.00011193291720701382, 0.0001984412083402276, 0.0001950853766174987, 0.000164438781212084, 0.00014093428035266697, 0.00018844140868168324, 0.00017189000209327787, 0.00016026418597903103, 0.00013655464863404632, 0.00018111034296453, 0.00014381477376446128, 0.0001345233031315729, 0.00013078533811494708, 0.00014274133718572557, 0.00014392216689884663, 0.00011242362961638719, 0.00010264343291055411, 0.0001324669283349067, 0.00011476465442683548, 0.00010308727360097691, 9.641810174798593e-05, 0.00010602219845168293, 0.00010317425767425448, 9.570685506332666e-05, 8.980527491075918e-05, 8.832637104205787e-05, 8.974187221610919e-05, 9.015315299620852e-05, 7.656779052922502e-05, 8.714834257261828e-05, 7.817049481673166e-05, 7.527579873567447e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.14759047608822584, "first_to_last_ratio": 1.9719337947467823, "imbalance_abs_log_ratio": 0.6790146829732503, "last_quartile_mean": 0.07484555337578058, "mean": 0.1104847895912826, "normalized": [ 0.8984334974418968, 1.5927952138130326, 1.565859646322909, 1.3198735575362357, 1.1312139236939245, 1.5125315007911495, 1.3796810264894532, 1.286366010451999, 1.0960606168617568, 1.4536884922319049, 1.1543342013148132, 1.0797558953237962, 1.0497530983161607, 1.1457182562234747, 1.1551962948455465, 0.9023721940039697, 0.8238711061937815, 1.0632504861804335, 0.9211625151546727, 0.8274335809310764, 0.7739032642161865, 0.8509909070635955, 0.8281317391228538, 0.7681943896830757, 0.7208251689669645, 0.7089546589522648, 0.7203162343708721, 0.7236173971219663, 0.6145740294939214, 0.6994991437375198, 0.6274381319743458, 0.6042038211744466 ], "peak_layer": 2, "peak_normalized": 1.5927952138130326, "population_cv": 0.28896658018674404, "rest_mean": 0.11481397047086998, "spike_contrast": 0.7586813040631772, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.08710721284151077, "top_five_layers": [ 2, 3, 6, 10, 7 ], "top_five_spike_overlap": 0 }, "values": [ 0.09926323592662811, 0.1759796440601349, 0.1730036735534668, 0.14582595229148865, 0.12498193234205246, 0.16711172461509705, 0.15243376791477203, 0.14212387800216675, 0.12109802663326263, 0.16061046719551086, 0.1275363713502884, 0.1192966029047966, 0.11598175019025803, 0.12658444046974182, 0.12763161957263947, 0.09969840198755264, 0.0910252258181572, 0.11747300624847412, 0.10177444666624069, 0.09141882508993149, 0.08550453931093216, 0.09402155131101608, 0.09149596095085144, 0.08487379550933838, 0.0796402171254158, 0.0783287063241005, 0.07958398759365082, 0.07994871586561203, 0.06790108233690262, 0.07728401571512222, 0.069322369992733, 0.06675533205270767 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 5.093040158499207e-05, "first_to_last_ratio": 0.8860093039525244, "imbalance_abs_log_ratio": 0.12102782735710801, "last_quartile_mean": 5.748291960117058e-05, "mean": 5.7093861016710434e-05, "normalized": [ 0.4337136137178416, 0.9185569667519601, 0.9444095926066538, 0.8728787927927439, 0.7849140301431445, 1.1296292591692207, 1.0726593944235638, 0.9796135302182263, 0.9386642177229508, 1.3046353648934683, 1.0461832118506482, 1.0032917361864164, 0.9662905330123907, 1.0760080966276753, 1.1828739682986207, 0.9087955578115051, 0.9194941496879511, 1.198292373503509, 1.0946132296674007, 1.0673695485587213, 0.9244260207272295, 1.0525679458300632, 1.125653431968099, 0.9999504891362015, 0.9629027708994021, 1.0063234987710272, 0.9975933861195996, 1.0791525148711678, 0.9412468232151083, 1.085173475571988, 1.0031556955576197, 0.9789667796878816 ], "peak_layer": 10, "peak_normalized": 1.3046353648934683, "population_cv": 0.1441687540611157, "rest_mean": 5.695535414647397e-05, "spike_contrast": 1.0155638391297448, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.784179811598733e-05, "top_five_layers": [ 10, 18, 15, 6, 23 ], "top_five_spike_overlap": 1 }, "values": [ 2.4762384782661684e-05, 5.2443963795667514e-05, 5.3919990023132414e-05, 4.9836020480142906e-05, 4.481377254705876e-05, 6.449489592341706e-05, 6.124226638348773e-05, 5.592991874436848e-05, 5.3591964388033375e-05, 7.448667020071298e-05, 5.973063889541663e-05, 5.728179894504137e-05, 5.516925739357248e-05, 6.143345672171563e-05, 6.75348419463262e-05, 5.1886647270293906e-05, 5.249747118796222e-05, 6.84151382301934e-05, 6.249569560168311e-05, 6.094024865888059e-05, 5.277905074763112e-05, 6.009516800986603e-05, 6.426790059776977e-05, 5.7091034250333905e-05, 5.4975836974335834e-05, 5.74548939766828e-05, 5.695645813830197e-05, 6.161298369988799e-05, 5.373941530706361e-05, 6.19567435933277e-05, 5.727403186028823e-05, 5.58929932594765e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 5.801741053801379e-05, "first_to_last_ratio": 0.954195857595535, "imbalance_abs_log_ratio": 0.046886327142486896, "last_quartile_mean": 6.08024129178375e-05, "mean": 6.296396202287724e-05, "normalized": [ 0.4796364558374551, 0.9568353419130547, 0.9804638447804884, 0.8949545354119591, 0.8237377234766136, 1.1597846916175047, 1.085649359633547, 0.9904450076993521, 0.9595078978373608, 1.2941748206155441, 1.0680753760655664, 1.0231559828609242, 0.9904019047543832, 1.096490963646882, 1.1850351594596897, 0.9422215946487361, 0.9418631353047864, 1.1705024179981143, 1.0926353878857977, 1.0382275689133567, 0.9320833313178482, 1.0784583323883348, 1.0985080196909318, 0.9917906747078287, 0.9476388143731139, 0.9855066582268529, 0.978396290111904, 1.028407839812513, 0.8982055143685478, 1.0282804954549911, 0.9451124964564701, 0.9138123627295472 ], "peak_layer": 10, "peak_normalized": 1.2941748206155441, "population_cv": 0.13431233496292436, "rest_mean": 6.285090866084729e-05, "spike_contrast": 1.0115120295379672, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.357445017783903e-05, "top_five_layers": [ 10, 15, 18, 6, 23 ], "top_five_spike_overlap": 1 }, "values": [ 3.019981159013696e-05, 6.0246144130360335e-05, 6.173388828756288e-05, 5.634988337988034e-05, 5.186579073779285e-05, 7.302463927771896e-05, 6.835678505012766e-05, 6.236234185053036e-05, 6.041441884008236e-05, 8.148637425620109e-05, 6.725025741616264e-05, 6.442195444833487e-05, 6.235962791834027e-05, 6.903941539349034e-05, 7.461450877599418e-05, 5.9326004702597857e-05, 5.930343468207866e-05, 7.369946979451925e-05, 6.879665306769311e-05, 6.537092122016475e-05, 5.8687659475253895e-05, 6.790400948375463e-05, 6.916641723364592e-05, 6.244707037694752e-05, 5.966709431959316e-05, 6.205140380188823e-05, 6.160370685392991e-05, 6.475263216998428e-05, 5.655457789544016e-05, 6.474461406469345e-05, 5.950802733423188e-05, 5.753724690293893e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 1.7888660295284353e-05, "first_to_last_ratio": 0.41855462748958633, "imbalance_abs_log_ratio": 0.8709478659353102, "last_quartile_mean": 4.2739129185065394e-05, "mean": 3.1986895351110434e-05, "normalized": [ 0.2593955501951102, 0.4997829486650695, 0.48480994473028793, 0.4968760363401999, 0.5474125689219601, 0.7788406177150305, 0.7554473604850604, 0.6514322393717875, 0.7578135841604356, 0.9674810188795085, 0.9032772556875033, 0.9019220651946442, 0.8311356963419073, 0.9587970177975769, 1.1401394869948454, 0.8953886492354074, 1.0262272842074007, 1.1413916918814533, 1.174037958462901, 1.194927715786966, 1.0267140632187253, 1.3434898267078448, 1.3729067467361273, 1.2011929483563188, 1.1735959904166378, 1.3266745689353452, 1.2739952988364465, 1.4049535244931144, 1.2752889026856997, 1.4680979746684197, 1.4182292846928506, 1.3483241791974143 ], "peak_layer": 30, "peak_normalized": 1.4680979746684197, "population_cv": 0.3184955307507873, "rest_mean": 3.066251991657299e-05, "spike_contrast": 1.276428773763581, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.913852269761264e-05, "top_five_layers": [ 30, 31, 28, 23, 32 ], "top_five_spike_overlap": 1 }, "values": [ 8.297258318634704e-06, 1.5986504877218977e-05, 1.5507564967265353e-05, 1.589352177688852e-05, 1.7510028555989265e-05, 2.491269333404489e-05, 2.416441566310823e-05, 2.083729486912489e-05, 2.4240103812189773e-05, 3.094671410508454e-05, 2.889303505071439e-05, 2.8849686714238487e-05, 2.6585450541460887e-05, 3.066893987124786e-05, 3.6469522456172854e-05, 2.8640703021665104e-05, 3.282582474639639e-05, 3.6509576602838933e-05, 3.755382931558415e-05, 3.822202779701911e-05, 3.2841395295690745e-05, 4.2974068492185324e-05, 4.391502443468198e-05, 3.842243313556537e-05, 3.7539692129939795e-05, 4.243620060151443e-05, 4.0751154301688075e-05, 4.494010136113502e-05, 4.0792532672639936e-05, 4.695989628089592e-05, 4.5364751713350415e-05, 4.3128704419359565e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00022091567916504573, "first_to_last_ratio": 1.232986382535498, "imbalance_abs_log_ratio": 0.20943917994887448, "last_quartile_mean": 0.00017917122386279516, "mean": 0.0002198539991695725, "normalized": [ 0.4672441326905094, 1.0299890566632133, 1.08986441571188, 1.0084939820045402, 0.8814237539069011, 1.2693826047962031, 1.1886514905905403, 1.1035827467935222, 1.0473307057033197, 1.4274044616309502, 1.1442998336482333, 1.0911283607583675, 1.1094038041384326, 1.1946873974128753, 1.2511466100570947, 0.9923165311956169, 0.9239703707694655, 1.2473578196113966, 1.1181336030529359, 1.0160745414007064, 0.9397394354798955, 0.9905446518023159, 1.0121913654482901, 0.9359943959598221, 0.8839446941554127, 0.8510229499852975, 0.8648513533654478, 0.8786898836618169, 0.7455559493639876, 0.8399702482950041, 0.7392957942269532, 0.7163130557190521 ], "peak_layer": 10, "peak_normalized": 1.4274044616309502, "population_cv": 0.18952709251819305, "rest_mean": 0.0002217885959436427, "spike_contrast": 0.9441746799407325, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00020940717658959329, "top_five_layers": [ 10, 6, 15, 18, 14 ], "top_five_spike_overlap": 0 }, "values": [ 0.00010272549116052687, 0.00022644721320830286, 0.00023961105034686625, 0.00022172143508214504, 0.0001937845372594893, 0.0002790788421407342, 0.0002613297838252038, 0.00024262708029709756, 0.00023025984410196543, 0.000313820579322055, 0.00025157889467664063, 0.00023988893372006714, 0.00024390686303377151, 0.000262656802078709, 0.00027506958576850593, 0.0002181647578254342, 0.00020313858112785965, 0.00027423660503700376, 0.0002458261442370713, 0.00022338805138133466, 0.00020660547306761146, 0.00021777520305477083, 0.00022253431961871684, 0.00020578211115207523, 0.0001943387760547921, 0.00018710079893935472, 0.0001901410287246108, 0.00019318348495289683, 0.00016391345707233995, 0.00018467081827111542, 0.00016253713693004102, 0.00015748428995721042 ] } } }, "pre_mlp_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "2999a257265024350fd2665e2a8d452a03cd662193b4245d15290408ad29922d", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.014061377383768559, 0.014165290631353855, 0.014281988143920898, 0.014350161887705326, 0.00966146495193243, 0.009741888381540775, 0.009820535778999329, 0.00991273857653141, 0.0074927168898284435, 0.007546746172010899, 0.00766373984515667, 0.007762390188872814, 0.006325984839349985, 0.006375366821885109, 0.006445344537496567, 0.006555322092026472, 0.005531130358576775, 0.005611324217170477, 0.0056777093559503555, 0.00574630219489336, 0.004946626722812653, 0.0050392793491482735, 0.005084683652967215, 0.005203457549214363, 0.004667261149734259, 0.004717131145298481, 0.004768412560224533, 0.004826710093766451, 0.0044052014127373695, 0.004504534415900707, 0.004574235528707504, 0.004644096828997135 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 3.729895206561196e-05, "first_to_last_ratio": 1.2469721805352876, "imbalance_abs_log_ratio": 0.22071835733540474, "last_quartile_mean": 2.9911615229138988e-05, "mean": 3.340667609563752e-05, "normalized": [ 1.308752778077389, 1.2456464596610868, 1.0236956931411572, 0.9922285647386262, 1.26903181062375, 1.1170846373362797, 1.0287316533628064, 0.9469237624012169, 1.1300261751959062, 1.1132698789554223, 1.0126240729566007, 0.9379202574439188, 1.1130503370514597, 1.070713488608195, 0.9897642722064988, 0.922085688720865, 1.036831181462568, 0.9898105546019274, 0.9395130252451869, 0.9001967292772246, 0.9975952535130206, 0.9404117749145336, 0.9224862220158919, 0.8885787590271026, 0.9461670180110204, 0.8974372095121397, 0.8904691498559152, 0.8684437934704528, 0.9244147277585869, 0.9015689206268677, 0.8740901912628669, 0.8604359589635168 ], "peak_layer": 1, "peak_normalized": 1.308752778077389, "population_cv": 0.1164905978879517, "rest_mean": 3.378375206213807e-05, "spike_contrast": 0.9285666618330386, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.137046587653458e-05, "top_five_layers": [ 1, 5, 2, 9, 6 ], "top_five_spike_overlap": 0 }, "values": [ 4.3721080146497115e-05, 4.161290780757554e-05, 3.419827044126578e-05, 3.314705827506259e-05, 4.239413465256803e-05, 3.73180846509058e-05, 3.4366505133220926e-05, 3.1633575417799875e-05, 3.7750418414361775e-05, 3.719064625329338e-05, 3.3828404411906376e-05, 3.1332798243965954e-05, 3.718331208801828e-05, 3.5768978705164045e-05, 3.306473445263691e-05, 3.080381793552078e-05, 3.463708344497718e-05, 3.3066280593629926e-05, 3.138600732199848e-05, 3.007258055731654e-05, 3.332634150865488e-05, 3.1416031561093405e-05, 3.0817198421573266e-05, 2.9684462788281962e-05, 3.1608295103069395e-05, 2.9980394174344838e-05, 2.9747614462394267e-05, 2.9011820515734144e-05, 3.0881623388268054e-05, 3.01184209092753e-05, 2.9200447897892445e-05, 2.874430538213346e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 6.705871965095866e-05, "first_to_last_ratio": 0.7180297289018083, "imbalance_abs_log_ratio": 0.33124430563536084, "last_quartile_mean": 9.33926785364747e-05, "mean": 8.228966771639534e-05, "normalized": [ 0.8685826408392191, 0.8249006353131848, 0.7154905339803773, 0.6884730939760333, 0.9470487095509537, 0.8638069620260335, 0.8256155016985178, 0.7853664463613442, 0.9851686390305121, 0.9692511243547932, 0.9083804947955202, 0.8729855457896263, 1.0422548534804916, 1.0255492616133721, 0.9871528463964067, 0.9398223255303129, 1.0958375595009802, 1.0726325633807163, 1.0495259774381736, 1.0230838788771643, 1.1576441025653552, 1.1142532603054807, 1.1008567438213033, 1.056908774400763, 1.159063578749664, 1.1192434432436902, 1.1032241584740294, 1.1060435702772593, 1.1786235078649367, 1.1628365878275773, 1.141840825041133, 1.1085318534950752 ], "peak_layer": 29, "peak_normalized": 1.1786235078649367, "population_cv": 0.1341844256762087, "rest_mean": 8.049536754023629e-05, "spike_contrast": 1.1426606459269575, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.197888866765424e-05, "top_five_layers": [ 29, 30, 25, 21, 31 ], "top_five_spike_overlap": 2 }, "values": [ 7.14753768988885e-05, 6.788079917896539e-05, 5.887747829547152e-05, 5.66542221349664e-05, 7.793232362018898e-05, 7.108238787623122e-05, 6.793962529627606e-05, 6.462754390668124e-05, 8.106919995043427e-05, 7.975935295689851e-05, 7.475032907677814e-05, 7.183769048424438e-05, 8.576680556870997e-05, 8.4392107964959e-05, 8.123247971525416e-05, 7.733766688033938e-05, 9.017610864248127e-05, 8.826657722238451e-05, 8.636514394311234e-05, 8.418923243880272e-05, 9.526214853394777e-05, 9.169153054244816e-05, 9.0589135652408e-05, 8.697267185198143e-05, 9.537895675748587e-05, 9.210217103827745e-05, 9.078394941752777e-05, 9.101595787797123e-05, 9.698853682493791e-05, 9.56894364207983e-05, 9.396170207764953e-05, 9.122071787714958e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.05946834245696664, "first_to_last_ratio": 0.7180297222365345, "imbalance_abs_log_ratio": 0.33124431491808765, "last_quartile_mean": 0.08282156102359295, "mean": 0.07297530013602227, "normalized": [ 0.8685826122359729, 0.8249005917343738, 0.71549052911611, 0.6884731350635278, 0.9470486699630107, 0.8638070113577097, 0.8256154769796886, 0.7853664269052657, 0.9851686341386658, 0.9692511556795396, 0.9083805454214465, 0.8729855579949275, 1.0422548040385637, 1.0255492365750694, 0.9871528950273681, 0.9398223256645052, 1.0958375053304956, 1.07263252629438, 1.0495259690486791, 1.0230837911446407, 1.1576441435704576, 1.1142533045483842, 1.1008568124470575, 1.0569088284972943, 1.1590636022344982, 1.119243411839159, 1.1032241423789504, 1.1060435591240758, 1.1786235037173105, 1.1628366076958818, 1.141840809884862, 1.1085318743481278 ], "peak_layer": 29, "peak_normalized": 1.1786235037173105, "population_cv": 0.13418442668798974, "rest_mean": 0.07138409617322462, "spike_contrast": 1.1426607032635474, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.08156780153512955, "top_five_layers": [ 29, 30, 25, 21, 31 ], "top_five_spike_overlap": 2 }, "values": [ 0.06338507682085037, 0.0601973682641983, 0.05221313610672951, 0.050241533666849136, 0.0691111609339714, 0.06303657591342926, 0.060249537229537964, 0.05731235072016716, 0.07189297676086426, 0.07073139399290085, 0.06628934293985367, 0.06370638310909271, 0.07605885714292526, 0.0748397633433342, 0.07203777879476547, 0.06858381628990173, 0.07996907085180283, 0.07827568054199219, 0.07658947259187698, 0.07465984672307968, 0.0844794288277626, 0.08131296932697296, 0.0803353562951088, 0.07712823897600174, 0.08458301424980164, 0.08167712390422821, 0.08050811290740967, 0.08071386069059372, 0.0860104039311409, 0.08485835045576096, 0.08332617580890656, 0.08089544624090195 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 7.408341275549901e-06, "first_to_last_ratio": 0.49124437919492936, "imbalance_abs_log_ratio": 0.7108135577065074, "last_quartile_mean": 1.5080765479069669e-05, "mean": 1.1844782648040564e-05, "normalized": [ 0.5657990507718705, 0.5382233871697658, 0.49942145823009426, 0.48906984415931193, 0.8062921756393574, 0.7122953827323945, 0.7013508394120195, 0.6911626993599347, 0.8940675881372303, 0.868722127857376, 0.9735911331661544, 0.854855860776803, 0.989483587379368, 1.0059088486455374, 1.0303729006301217, 0.9941025538400318, 1.1620912657632405, 1.127012997009919, 1.113401270263767, 1.066097770938846, 1.1826034542812638, 1.1643547935214302, 1.2049778184223279, 1.1791491538213539, 1.2365172470752577, 1.219921447624247, 1.2337761202582145, 1.2288278254433027, 1.3039217543779833, 1.3631114810840155, 1.2856635723507386, 1.3138525898567213 ], "peak_layer": 30, "peak_normalized": 1.3631114810840155, "population_cv": 0.2549101540724733, "rest_mean": 1.1420299651271543e-05, "spike_contrast": 1.237882653019464, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.4136990830593277e-05, "top_five_layers": [ 30, 32, 29, 31, 25 ], "top_five_spike_overlap": 1 }, "values": [ 6.701766778860474e-06, 6.3751390371180605e-06, 5.915538622502936e-06, 5.7929260037781205e-06, 9.550355571263935e-06, 8.436983989668079e-06, 8.307348252856173e-06, 8.186671948351432e-06, 1.0590036254143342e-05, 1.0289824786013924e-05, 1.1531975360412616e-05, 1.0125581866304856e-05, 1.1720218026312068e-05, 1.1914771675947122e-05, 1.220454305439489e-05, 1.1774928680097219e-05, 1.3764718460151926e-05, 1.334922399109928e-05, 1.3187996046326589e-05, 1.2627696378331166e-05, 1.4007680874783546e-05, 1.379152945446549e-05, 1.4272700354922563e-05, 1.3966765436634887e-05, 1.4646278032159898e-05, 1.4449704394792207e-05, 1.4613809980801307e-05, 1.4555198504240252e-05, 1.5444669770658948e-05, 1.614575921848882e-05, 1.5228405572997872e-05, 1.5562298358418047e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 5.6029601637419546e-05, "first_to_last_ratio": 0.6070213107493427, "imbalance_abs_log_ratio": 0.49919138022039955, "last_quartile_mean": 9.230252817360451e-05, "mean": 7.840498903988191e-05, "normalized": [ 0.6598133090619259, 0.6419677920829977, 0.6005256373772865, 0.5913463573822231, 0.8562376513989928, 0.8062968559250673, 0.7923156566726156, 0.768439227921366, 0.9790122729268586, 0.9665730330981341, 0.9163710065350301, 0.8881686280668529, 1.0631190324760253, 1.0511030519715439, 1.0188080221730096, 0.9712812246823179, 1.1361272291219966, 1.1138561419734825, 1.0918134348286874, 1.0639217496772315, 1.203414696914147, 1.1592426077267617, 1.1442664103435203, 1.097952968622282, 1.2042281788778202, 1.1625445130141736, 1.1452986211445626, 1.1473932954411665, 1.2233942816523278, 1.205318760675343, 1.1829469858778916, 1.146901364356358 ], "peak_layer": 29, "peak_normalized": 1.2233942816523278, "population_cv": 0.1898597249088972, "rest_mean": 7.605543874820728e-05, "spike_contrast": 1.1977126437531995, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.109256061492487e-05, "top_five_layers": [ 29, 30, 25, 21, 31 ], "top_five_spike_overlap": 2 }, "values": [ 5.173265526536852e-05, 5.033347770222463e-05, 4.708420601673424e-05, 4.636450466932729e-05, 6.713330367347226e-05, 6.321769615169615e-05, 6.212150037754327e-05, 6.024946924299002e-05, 7.675944652874023e-05, 7.578414806630462e-05, 7.184805872384459e-05, 6.963685154914856e-05, 8.335383608937263e-05, 8.241172326961532e-05, 7.987963181221858e-05, 7.615329377586022e-05, 8.907804294722155e-05, 8.733187860343605e-05, 8.560362039133906e-05, 8.341677312273532e-05, 9.435371612198651e-05, 9.089040395338088e-05, 8.971619536168873e-05, 8.608499047113582e-05, 9.441749716643244e-05, 9.114928980125114e-05, 8.97971258382313e-05, 8.996135875349864e-05, 9.592021524440497e-05, 9.450300422031432e-05, 9.274894546251744e-05, 8.992278890218586e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 4.8511469685763586e-05, "first_to_last_ratio": 0.5271352332510719, "imbalance_abs_log_ratio": 0.6402981537748061, "last_quartile_mean": 9.202850924339145e-05, "mean": 7.532146378252946e-05, "normalized": [ 0.548104082267993, 0.5416830199005969, 0.528884702689083, 0.5247448675895723, 0.7777813723908718, 0.7487613756360456, 0.7496059866168319, 0.7329069161583202, 0.9480676391902623, 0.9399123881532756, 0.9000936195631334, 0.883441930780847, 1.0560589316963704, 1.0534552095740957, 1.032095781768693, 0.9835620691098637, 1.1601754560133901, 1.1467788545782247, 1.1233340534265408, 1.0967375255670042, 1.2396083970871608, 1.1951467072205837, 1.1828066060767042, 1.1317734013234224, 1.244698956870482, 1.2025862582796845, 1.1881970083126314, 1.1882256015362047, 1.2689460104605979, 1.2589162611011824, 1.2307315535498626, 1.1921774555104672 ], "peak_layer": 29, "peak_normalized": 1.2689460104605979, "population_cv": 0.23486334953575583, "rest_mean": 7.254842300123225e-05, "spike_contrast": 1.244629176846487, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.029588400153444e-05, "top_five_layers": [ 29, 30, 25, 21, 31 ], "top_five_spike_overlap": 2 }, "values": [ 4.1284001781605184e-05, 4.080035796505399e-05, 3.9836369978729635e-05, 3.952455153921619e-05, 5.858363147126511e-05, 5.639780283672735e-05, 5.6461420172126964e-05, 5.520362174138427e-05, 7.140984234865755e-05, 7.079557690303773e-05, 6.77963689668104e-05, 6.654213939327747e-05, 7.954390457598493e-05, 7.934778841445222e-05, 7.773896504659206e-05, 7.408333476632833e-05, 8.738611359149218e-05, 8.637706196168438e-05, 8.461116522084922e-05, 8.26078758109361e-05, 9.336911898571998e-05, 9.002019942272455e-05, 8.909072494134307e-05, 8.524682925781235e-05, 9.375254740007222e-05, 9.058055729838088e-05, 8.949673792812973e-05, 8.949889161158353e-05, 9.557887096889317e-05, 9.482341556577012e-05, 9.270050213672221e-05, 8.979655103757977e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 9.61669284151867e-05, "first_to_last_ratio": 0.8297833624837898, "imbalance_abs_log_ratio": 0.18659062131596024, "last_quartile_mean": 0.00011589401856326731, "mean": 0.00010716335873439675, "normalized": [ 0.9380742209455786, 0.8898820858744699, 0.7837417114644826, 0.7583102664636657, 1.0822070619439073, 0.9520749081713696, 0.9107982467806989, 0.8640017807748708, 1.0669538303069324, 1.0266950649381863, 0.9594029855154438, 0.8901506143407661, 1.0795289063552718, 1.0379165674077444, 0.9706971366471856, 0.914979822300863, 1.052939971267624, 1.0225707935964177, 0.9851238030379684, 0.9636652893153618, 1.0990044510799908, 1.0523904893769196, 1.0422234788441622, 1.0049019599938585, 1.0962247910521359, 1.0599378706264646, 1.05039930496957, 1.0583562345651827, 1.1199788667078585, 1.1100529546792268, 1.0850973923636869, 1.0717171382921347 ], "peak_layer": 29, "peak_normalized": 1.1199788667078585, "population_cv": 0.09156717959150794, "rest_mean": 0.00010599351123716958, "spike_contrast": 1.0706366257223145, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00011348053521942347, "top_five_layers": [ 29, 30, 21, 25, 31 ], "top_five_spike_overlap": 2 }, "values": [ 0.00010052718425868079, 9.536275319987908e-05, 8.398839418077841e-05, 8.12630751170218e-05, 0.00011597294360399246, 0.00010202754492638633, 9.760419925441965e-05, 9.25893327803351e-05, 0.00011433835607022047, 0.00011002409155480564, 0.00010281284630764276, 9.539152961224318e-05, 0.000115685943455901, 0.0001112266254494898, 0.0001040231654769741, 9.805231093196198e-05, 0.00011283658386673778, 0.00010958212078548968, 0.00010556917550275102, 0.00010326960909878835, 0.00011777300824178383, 0.0001127776995417662, 0.00011168816854478791, 0.00010768866923172027, 0.00011747513053705916, 0.00011358650226611644, 0.00011256431753281504, 0.00011341700883349404, 0.00012002069706795737, 0.00011895700299646705, 0.00011628268111962825, 0.00011484880815260112 ] } } } }, "selector": { "expected_uniform_count": 57, "identities_unique": true, "identity_order_sha256": "97fd9721a263024f6a746638f32e0ad9a988cd4c111c00e9e178c1022e5c9d5e", "passed": true, "uniform_count": 57, "uniform_indices": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 56, 57, 58, 59, 60, 61, 62, 63, 64 ], "visit_count": 65, "visits": [ { "branch": "attention", "coefficient": "uniform", "group": 1, "index": 0, "kind": "depth", "layer": 1, "offset": 1, "sources": 1, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 1, "index": 1, "kind": "depth", "layer": 1, "offset": 1, "sources": 2, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 1, "index": 2, "kind": "depth", "layer": 2, "offset": 2, "sources": 2, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 1, "index": 3, "kind": "depth", "layer": 2, "offset": 2, "sources": 2, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 1, "index": 4, "kind": "depth", "layer": 3, "offset": 3, "sources": 2, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 1, "index": 5, "kind": "depth", "layer": 3, "offset": 3, "sources": 2, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 1, "index": 6, "kind": "depth", "layer": 4, "offset": 4, "sources": 2, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 1, "index": 7, "kind": "depth", "layer": 4, "offset": 4, "sources": 2, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 2, "index": 8, "kind": "depth", "layer": 5, "offset": 1, "sources": 2, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 2, "index": 9, "kind": "depth", "layer": 5, "offset": 1, "sources": 3, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 2, "index": 10, "kind": "depth", "layer": 6, "offset": 2, "sources": 3, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 2, "index": 11, "kind": "depth", "layer": 6, "offset": 2, "sources": 3, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 2, "index": 12, "kind": "depth", "layer": 7, "offset": 3, "sources": 3, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 2, "index": 13, "kind": "depth", "layer": 7, "offset": 3, "sources": 3, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 2, "index": 14, "kind": "depth", "layer": 8, "offset": 4, "sources": 3, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 2, "index": 15, "kind": "depth", "layer": 8, "offset": 4, "sources": 3, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 3, "index": 16, "kind": "depth", "layer": 9, "offset": 1, "sources": 3, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 3, "index": 17, "kind": "depth", "layer": 9, "offset": 1, "sources": 4, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 3, "index": 18, "kind": "depth", "layer": 10, "offset": 2, "sources": 4, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 3, "index": 19, "kind": "depth", "layer": 10, "offset": 2, "sources": 4, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 3, "index": 20, "kind": "depth", "layer": 11, "offset": 3, "sources": 4, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 3, "index": 21, "kind": "depth", "layer": 11, "offset": 3, "sources": 4, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 3, "index": 22, "kind": "depth", "layer": 12, "offset": 4, "sources": 4, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 3, "index": 23, "kind": "depth", "layer": 12, "offset": 4, "sources": 4, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 4, "index": 24, "kind": "depth", "layer": 13, "offset": 1, "sources": 4, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 4, "index": 25, "kind": "depth", "layer": 13, "offset": 1, "sources": 5, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 4, "index": 26, "kind": "depth", "layer": 14, "offset": 2, "sources": 5, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 4, "index": 27, "kind": "depth", "layer": 14, "offset": 2, "sources": 5, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 4, "index": 28, "kind": "depth", "layer": 15, "offset": 3, "sources": 5, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 4, "index": 29, "kind": "depth", "layer": 15, "offset": 3, "sources": 5, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 4, "index": 30, "kind": "depth", "layer": 16, "offset": 4, "sources": 5, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 4, "index": 31, "kind": "depth", "layer": 16, "offset": 4, "sources": 5, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 5, "index": 32, "kind": "depth", "layer": 17, "offset": 1, "sources": 5, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 5, "index": 33, "kind": "depth", "layer": 17, "offset": 1, "sources": 6, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 5, "index": 34, "kind": "depth", "layer": 18, "offset": 2, "sources": 6, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 5, "index": 35, "kind": "depth", "layer": 18, "offset": 2, "sources": 6, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 5, "index": 36, "kind": "depth", "layer": 19, "offset": 3, "sources": 6, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 5, "index": 37, "kind": "depth", "layer": 19, "offset": 3, "sources": 6, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 5, "index": 38, "kind": "depth", "layer": 20, "offset": 4, "sources": 6, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 5, "index": 39, "kind": "depth", "layer": 20, "offset": 4, "sources": 6, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 6, "index": 40, "kind": "depth", "layer": 21, "offset": 1, "sources": 6, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 6, "index": 41, "kind": "depth", "layer": 21, "offset": 1, "sources": 7, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 6, "index": 42, "kind": "depth", "layer": 22, "offset": 2, "sources": 7, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 6, "index": 43, "kind": "depth", "layer": 22, "offset": 2, "sources": 7, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 6, "index": 44, "kind": "depth", "layer": 23, "offset": 3, "sources": 7, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 6, "index": 45, "kind": "depth", "layer": 23, "offset": 3, "sources": 7, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 6, "index": 46, "kind": "depth", "layer": 24, "offset": 4, "sources": 7, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 6, "index": 47, "kind": "depth", "layer": 24, "offset": 4, "sources": 7, "use_uniform": true }, { "branch": "attention", "coefficient": "detached_learned", "group": 7, "index": 48, "kind": "depth", "layer": 25, "offset": 1, "sources": 7, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 7, "index": 49, "kind": "depth", "layer": 25, "offset": 1, "sources": 8, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 7, "index": 50, "kind": "depth", "layer": 26, "offset": 2, "sources": 8, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 7, "index": 51, "kind": "depth", "layer": 26, "offset": 2, "sources": 8, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 7, "index": 52, "kind": "depth", "layer": 27, "offset": 3, "sources": 8, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 7, "index": 53, "kind": "depth", "layer": 27, "offset": 3, "sources": 8, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 7, "index": 54, "kind": "depth", "layer": 28, "offset": 4, "sources": 8, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 7, "index": 55, "kind": "depth", "layer": 28, "offset": 4, "sources": 8, "use_uniform": false }, { "branch": "attention", "coefficient": "uniform", "group": 8, "index": 56, "kind": "depth", "layer": 29, "offset": 1, "sources": 8, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 8, "index": 57, "kind": "depth", "layer": 29, "offset": 1, "sources": 9, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 8, "index": 58, "kind": "depth", "layer": 30, "offset": 2, "sources": 9, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 8, "index": 59, "kind": "depth", "layer": 30, "offset": 2, "sources": 9, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 8, "index": 60, "kind": "depth", "layer": 31, "offset": 3, "sources": 9, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 8, "index": 61, "kind": "depth", "layer": 31, "offset": 3, "sources": 9, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 8, "index": 62, "kind": "depth", "layer": 32, "offset": 4, "sources": 9, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 8, "index": 63, "kind": "depth", "layer": 32, "offset": 4, "sources": 9, "use_uniform": true }, { "branch": "output", "coefficient": "uniform", "group": null, "index": 64, "kind": "output", "layer": null, "offset": null, "sources": 9, "use_uniform": true } ], "visits_sha256": "9dc56a1d63831ead7802d3ab462a452e1dc050e7e8c2166997eba292fe80e060" } }, "uniform_except_groups_6_7": { "bits_per_byte": 7.959341947457508, "forward": { "activation_sha256": { "attention_branch_output": "676759a2d7c0615ad7e1f246e90927fb829d00c8b6c3745032ae6b27ab7d902a", "mlp_branch_output": "afe29f3ef4d2992353f0bb59fe7c5c2c5050fca671a53184825a8d63cd83a2bc", "post_attention_state": "31853ffb07b24cc59e774bb20656840b4358b3080b42516d380f2dc0f46404c8", "post_mlp_state": "7fd9c911e93175896ad618842c120e77b3f017c17b7eec9c2bfb01fc6dbc8ac2", "pre_attention_input": "de71d4cf2d509a1063f6eac7f50c4885e9a058dee7e4b993ad40f5527f528d00", "pre_mlp_input": "2999a257265024350fd2665e2a8d452a03cd662193b4245d15290408ad29922d" }, "logits_sha256": "873a309a197bfd08fa523edcb440d2697903a34abc02dad1c7f5991509c3c151", "loss_nats": 5.516995429992676, "mixer_summary_sha256": "8450b24bdc777885a3357dc7b8080e7410acbe96943686433da76f379295133a" }, "loss_nats": 5.516995429992676, "loss_scale": 1.0, "mixers": [ { "branch": "attention", "entropy_mean": -0.0, "group": 1, "latest_source_mass": 1.0, "layer": 1, "max_source_mass": 1.0, "mixer_index": 0, "normalized_entropy": 1.0, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 1.0, "median": 1.0, "p05": 1.0, "p95": 1.0 } ], "sources": 1 }, { "branch": "mlp", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 1, "max_source_mass": 0.5, "mixer_index": 1, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 2, "max_source_mass": 0.5, "mixer_index": 2, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 2, "max_source_mass": 0.5, "mixer_index": 3, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 3, "max_source_mass": 0.5, "mixer_index": 4, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 3, "max_source_mass": 0.5, "mixer_index": 5, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 4, "max_source_mass": 0.5, "mixer_index": 6, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 4, "max_source_mass": 0.5, "mixer_index": 7, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6931471824645996, "group": 2, "latest_source_mass": 0.5, "layer": 5, "max_source_mass": 0.5, "mixer_index": 8, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "completed_group_1", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 5, "max_source_mass": 0.3333333432674408, "mixer_index": 9, "normalized_entropy": 1.0000000180537645, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 6, "max_source_mass": 0.3333333432674408, "mixer_index": 10, "normalized_entropy": 1.0000000180537645, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 6, "max_source_mass": 0.3333333432674408, "mixer_index": 11, "normalized_entropy": 1.0000000180537645, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 7, "max_source_mass": 0.3333333432674408, "mixer_index": 12, "normalized_entropy": 1.0000000180537645, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 7, "max_source_mass": 0.3333333432674408, "mixer_index": 13, "normalized_entropy": 1.0000000180537645, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 8, "max_source_mass": 0.3333333432674408, "mixer_index": 14, "normalized_entropy": 1.0000000180537645, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 8, "max_source_mass": 0.3333333432674408, "mixer_index": 15, "normalized_entropy": 1.0000000180537645, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0986123085021973, "group": 3, "latest_source_mass": 0.3333333432674408, "layer": 9, "max_source_mass": 0.3333333432674408, "mixer_index": 16, "normalized_entropy": 1.0000000180537645, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_2", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 9, "max_source_mass": 0.25, "mixer_index": 17, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 10, "max_source_mass": 0.25, "mixer_index": 18, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 10, "max_source_mass": 0.25, "mixer_index": 19, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 11, "max_source_mass": 0.25, "mixer_index": 20, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 11, "max_source_mass": 0.25, "mixer_index": 21, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 12, "max_source_mass": 0.25, "mixer_index": 22, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 12, "max_source_mass": 0.25, "mixer_index": 23, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3862943649291992, "group": 4, "latest_source_mass": 0.25, "layer": 13, "max_source_mass": 0.25, "mixer_index": 24, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_3", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 13, "max_source_mass": 0.20000000298023224, "mixer_index": 25, "normalized_entropy": 1.0000000186840277, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 14, "max_source_mass": 0.20000000298023224, "mixer_index": 26, "normalized_entropy": 1.0000000186840277, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 14, "max_source_mass": 0.20000000298023224, "mixer_index": 27, "normalized_entropy": 1.0000000186840277, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 15, "max_source_mass": 0.20000000298023224, "mixer_index": 28, "normalized_entropy": 1.0000000186840277, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 15, "max_source_mass": 0.20000000298023224, "mixer_index": 29, "normalized_entropy": 1.0000000186840277, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 16, "max_source_mass": 0.20000000298023224, "mixer_index": 30, "normalized_entropy": 1.0000000186840277, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 16, "max_source_mass": 0.20000000298023224, "mixer_index": 31, "normalized_entropy": 1.0000000186840277, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.6094379425048828, "group": 5, "latest_source_mass": 0.20000000298023224, "layer": 17, "max_source_mass": 0.20000000298023224, "mixer_index": 32, "normalized_entropy": 1.0000000186840277, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_4", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 17, "max_source_mass": 0.1666666716337204, "mixer_index": 33, "normalized_entropy": 1.0000001451965654, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 18, "max_source_mass": 0.1666666716337204, "mixer_index": 34, "normalized_entropy": 1.0000001451965654, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 18, "max_source_mass": 0.1666666716337204, "mixer_index": 35, "normalized_entropy": 1.0000001451965654, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 19, "max_source_mass": 0.1666666716337204, "mixer_index": 36, "normalized_entropy": 1.0000001451965654, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 19, "max_source_mass": 0.1666666716337204, "mixer_index": 37, "normalized_entropy": 1.0000001451965654, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 20, "max_source_mass": 0.1666666716337204, "mixer_index": 38, "normalized_entropy": 1.0000001451965654, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 20, "max_source_mass": 0.1666666716337204, "mixer_index": 39, "normalized_entropy": 1.0000001451965654, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.791759729385376, "group": 6, "latest_source_mass": 0.1666666716337204, "layer": 21, "max_source_mass": 0.1666666716337204, "mixer_index": 40, "normalized_entropy": 1.0000001451965654, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_5", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 21, "max_source_mass": 0.1428571492433548, "mixer_index": 41, "normalized_entropy": 1.00000003408302, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 22, "max_source_mass": 0.1428571492433548, "mixer_index": 42, "normalized_entropy": 1.00000003408302, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 22, "max_source_mass": 0.1428571492433548, "mixer_index": 43, "normalized_entropy": 1.00000003408302, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 23, "max_source_mass": 0.1428571492433548, "mixer_index": 44, "normalized_entropy": 1.00000003408302, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 23, "max_source_mass": 0.1428571492433548, "mixer_index": 45, "normalized_entropy": 1.00000003408302, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 24, "max_source_mass": 0.1428571492433548, "mixer_index": 46, "normalized_entropy": 1.00000003408302, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 24, "max_source_mass": 0.1428571492433548, "mixer_index": 47, "normalized_entropy": 1.00000003408302, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9459102153778076, "group": 7, "latest_source_mass": 0.1428571492433548, "layer": 25, "max_source_mass": 0.1428571492433548, "mixer_index": 48, "normalized_entropy": 1.00000003408302, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_6", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 25, "max_source_mass": 0.125, "mixer_index": 49, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 26, "max_source_mass": 0.125, "mixer_index": 50, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 26, "max_source_mass": 0.125, "mixer_index": 51, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 27, "max_source_mass": 0.125, "mixer_index": 52, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 27, "max_source_mass": 0.125, "mixer_index": 53, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 28, "max_source_mass": 0.125, "mixer_index": 54, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 28, "max_source_mass": 0.125, "mixer_index": 55, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.079441547393799, "group": 8, "latest_source_mass": 0.125, "layer": 29, "max_source_mass": 0.125, "mixer_index": 56, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_7", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 29, "max_source_mass": 0.1111111119389534, "mixer_index": 57, "normalized_entropy": 1.0000000180537645, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 30, "max_source_mass": 0.1111111119389534, "mixer_index": 58, "normalized_entropy": 1.0000000180537645, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 30, "max_source_mass": 0.1111111119389534, "mixer_index": 59, "normalized_entropy": 1.0000000180537645, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 31, "max_source_mass": 0.1111111119389534, "mixer_index": 60, "normalized_entropy": 1.0000000180537645, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 31, "max_source_mass": 0.1111111119389534, "mixer_index": 61, "normalized_entropy": 1.0000000180537645, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 32, "max_source_mass": 0.1111111119389534, "mixer_index": 62, "normalized_entropy": 1.0000000180537645, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 32, "max_source_mass": 0.1111111119389534, "mixer_index": 63, "normalized_entropy": 1.0000000180537645, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "output", "entropy_mean": 2.1972246170043945, "group": null, "latest_source_mass": 0.1111111119389534, "layer": null, "max_source_mass": 0.1111111119389534, "mixer_index": null, "normalized_entropy": 1.0000000180537645, "offset": null, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_8", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 } ], "mode": "uniform_except_groups_6_7", "optimizer_state_after": "b82a349700eda58e2330a22ca2ee1f7fb1406efd47fd9aa306122b368a114192", "optimizer_state_before": "b82a349700eda58e2330a22ca2ee1f7fb1406efd47fd9aa306122b368a114192", "optimizer_state_unchanged": true, "positions": { "attention_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "676759a2d7c0615ad7e1f246e90927fb829d00c8b6c3745032ae6b27ab7d902a", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.0022265110164880753, 0.0021613112185150385, 0.0025601957459002733, 0.0025323182344436646, 0.0027253651060163975, 0.002904455177485943, 0.0029208383057266474, 0.00299624796025455, 0.0030937574338167906, 0.0033804215490818024, 0.0034370317589491606, 0.0038969130255281925, 0.004255065228790045, 0.004119740333408117, 0.003970506135374308, 0.004385598469525576, 0.0041999719105660915, 0.005232699681073427, 0.004664790350943804, 0.004898969549685717, 0.005131533369421959, 0.005069608334451914, 0.0051318020559847355, 0.0058372304774820805, 0.0059690922498703, 0.00554921617731452, 0.0058550527319312096, 0.006255472078919411, 0.006502739619463682, 0.006122888531535864, 0.007071305997669697, 0.006241494789719582 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.00015962477027642308, "first_to_last_ratio": 3.7699040072151497, "imbalance_abs_log_ratio": 1.3270495388586014, "last_quartile_mean": 4.234186599205714e-05, "mean": 8.617327489446325e-05, "normalized": [ 2.5428430303860874, 2.3222064178412967, 2.1011513502014654, 1.8911470095267005, 1.7004447966680725, 1.5353700031893112, 1.4229335846259188, 1.3028620492588128, 1.2049362956920768, 1.1705197150696323, 1.0773612761873523, 1.0133200843661747, 0.9906112963967714, 0.9272530070793641, 0.8550207003106389, 0.8185600560490035, 0.7703816559177782, 0.7173918665339755, 0.6760157258502337, 0.6514535263006792, 0.6184138115448249, 0.6053606047045522, 0.5838821473278404, 0.5697017460729616, 0.5457761369570343, 0.5276086337508958, 0.5104655265174319, 0.4951752820428687, 0.47527333481689216, 0.465789239220051, 0.45896414036514044, 0.45180594922816086 ], "peak_layer": 1, "peak_normalized": 2.5428430303860874, "population_cv": 0.5705341151299013, "rest_mean": 9.280180475562466e-05, "spike_contrast": 0.5428688997682254, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.0379213644191625e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00021912511147093028, 0.00020011213200632483, 0.00018106309289578348, 0.00016296633111778647, 0.00014653289690613747, 0.00013230786134954542, 0.0001226188469445333, 0.00011227188952034339, 0.0001038333066389896, 0.00010086751717608422, 9.283974941354245e-05, 8.732111018616706e-05, 8.536421955795959e-05, 7.990442827576771e-05, 7.367993384832516e-05, 7.053800072753802e-05, 6.63863102090545e-05, 6.182000652188435e-05, 5.825448897667229e-05, 5.6137883802875876e-05, 5.329074338078499e-05, 5.216590579948388e-05, 5.031503678765148e-05, 4.909306517220102e-05, 4.703131708083674e-05, 4.5465763832908124e-05, 4.3988486140733585e-05, 4.2670875700423494e-05, 4.095585973118432e-05, 4.013858415419236e-05, 3.955044303438626e-05, 3.893359826179221e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.00021442661636683624, "first_to_last_ratio": 1.843144421770586, "imbalance_abs_log_ratio": 0.6114730379455638, "last_quartile_mean": 0.0001163373926829081, "mean": 0.00015221274247778638, "normalized": [ 1.7741446897870732, 1.648461275666186, 1.5301464224684427, 1.4217121808823485, 1.3265065496562456, 1.2454420544341354, 1.1897998831173078, 1.1336248235930768, 1.0869089599614004, 1.0643003125599004, 1.0227030055526172, 0.9941765648180964, 0.9788958472869692, 0.9511102301475627, 0.9228060651723192, 0.9052418824754537, 0.8846484396889256, 0.865292475366091, 0.8497311645646503, 0.8388691935254817, 0.8256303537004397, 0.8176325372729673, 0.8076852907347736, 0.8000670639426607, 0.7892588692968658, 0.7814415503461742, 0.7731202151361662, 0.7664994567889658, 0.758490072461763, 0.7529625284402992, 0.7485174912671357, 0.7441725498875049 ], "peak_layer": 1, "peak_normalized": 1.7741446897870732, "population_cv": 0.27364872432853743, "rest_mean": 0.00015762320577373935, "spike_contrast": 0.7803181014868814, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012299624067964033, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00027004742878489196, 0.0002509168116375804, 0.0002329077833564952, 0.00021640271006617695, 0.00020191119983792305, 0.00018957215070258826, 0.0001811027032090351, 0.00017255214333999902, 0.00016544139361940324, 0.00016200006939470768, 0.00015566842921543866, 0.0001513263414381072, 0.00014900042151566595, 0.0001447710965294391, 0.00014046284195501357, 0.0001377893495373428, 0.00013465476513374597, 0.00013170854072086513, 0.00012933991092722863, 0.0001276865805266425, 0.00012567146040964872, 0.00012445409083738923, 0.00012293999316170812, 0.00012178040196886286, 0.00012013525702059269, 0.00011894536146428436, 0.00011767874821089208, 0.00011667098442558199, 0.00011545185407157987, 0.00011461049143690616, 0.00011393390013836324, 0.00011327254469506443 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.19015566632151604, "first_to_last_ratio": 1.8431444167877922, "imbalance_abs_log_ratio": 0.6114730352421437, "last_quartile_mean": 0.10316916275769472, "mean": 0.13498378219082952, "normalized": [ 1.7741447141792424, 1.6484611693507762, 1.5301464164964773, 1.421712099711509, 1.3265065258282553, 1.245442086656075, 1.1897999853143832, 1.1336247998774232, 1.0869090145314644, 1.0643003539468234, 1.0227030148964142, 0.9941765578539046, 0.978895844117059, 0.9511102293531465, 0.9228060470726628, 0.9052419251832852, 0.8846484748200404, 0.8652925213936268, 0.8497311905993837, 0.8388692019864461, 0.8256303029992552, 0.8176324960789627, 0.8076852727084025, 0.800067049482784, 0.789258877083647, 0.7814415614650243, 0.7731201949085273, 0.7664994759042474, 0.758490077799444, 0.7529625178867759, 0.7485174642151933, 0.7441725362993381 ], "peak_layer": 1, "peak_normalized": 1.7741447141792424, "population_cv": 0.2736487184871982, "rest_mean": 0.13978183573042904, "spike_contrast": 0.7803180757143806, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.10907429307699204, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.23948076367378235, 0.2225155234336853, 0.20654495060443878, 0.1919080764055252, 0.17905686795711517, 0.16811448335647583, 0.16060370206832886, 0.1530209630727768, 0.14671508967876434, 0.14366328716278076, 0.13804832100868225, 0.13419771194458008, 0.1321350634098053, 0.12838445603847504, 0.12456385046243668, 0.12219297885894775, 0.11941319704055786, 0.116800457239151, 0.11469992995262146, 0.11323373764753342, 0.11144670099020004, 0.11036712676286697, 0.10902441293001175, 0.10799607634544373, 0.10653714835643768, 0.10548193752765656, 0.10435868799686432, 0.1034649983048439, 0.10238385945558548, 0.10163772851228714, 0.10103771835565567, 0.10045122355222702 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 6.883365585963475e-05, "first_to_last_ratio": 2.2497546817252747, "imbalance_abs_log_ratio": 0.8108211799277664, "last_quartile_mean": 3.059607183786284e-05, "mean": 4.691841348858361e-05, "normalized": [ 1.7359303646705315, 1.6475859007944922, 1.5536416135247637, 1.4921074478374423, 1.436565604934822, 1.343834173349448, 1.2787439414688093, 1.2483315192759759, 1.191227587726315, 1.1603086071531024, 1.0995306735006558, 1.059977946939621, 1.0583984121521266, 1.0176934649371145, 0.9754909410255406, 0.9631621800279858, 0.9242301483750994, 0.8752327792179452, 0.8454160845678106, 0.8232909667825862, 0.7863453139125789, 0.77746685672799, 0.7497386589361914, 0.7388508716017261, 0.7093180450413769, 0.6908354487677191, 0.6685471901846501, 0.6592354484095886, 0.631440334976849, 0.6235901145414114, 0.6199506557967245, 0.6139807028410055 ], "peak_layer": 1, "peak_normalized": 1.7359303646705315, "population_cv": 0.3220412433027962, "rest_mean": 4.907019663530971e-05, "spike_contrast": 0.7193528234378936, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.529878449626267e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 8.144709863699973e-05, 7.730211655143648e-05, 7.289439963642508e-05, 7.000731420703232e-05, 6.740137905580923e-05, 6.305056740529835e-05, 5.999663699185476e-05, 5.856973439222202e-05, 5.5890508519951254e-05, 5.4439839004771784e-05, 5.1588234782684594e-05, 4.9732483603293076e-05, 4.965837433701381e-05, 4.774856279254891e-05, 4.576848732540384e-05, 4.519004141911864e-05, 4.3363412260077894e-05, 4.106453343410976e-05, 3.966558142565191e-05, 3.862750600092113e-05, 3.6894074582960457e-05, 3.647751145763323e-05, 3.517654840834439e-05, 3.4665710700210184e-05, 3.328007733216509e-05, 3.241290323785506e-05, 3.136717350571416e-05, 3.0930281354812905e-05, 2.9626178729813546e-05, 2.925785884144716e-05, 2.9087101211189292e-05, 2.8807000489905477e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 0.0001734125216898974, "first_to_last_ratio": 1.495517223041696, "imbalance_abs_log_ratio": 0.4024721155990803, "last_quartile_mean": 0.00011595488103921525, "mean": 0.00013990532806928968, "normalized": [ 1.3768889945483427, 1.3340940553978193, 1.2910049686488183, 1.2527203273458163, 1.2147536525239127, 1.1769854337196524, 1.14744939519443, 1.1220955988146075, 1.0944022472959734, 1.0773022658134643, 1.0508559212107247, 1.03206209473486, 1.0208171898483585, 1.000561882500425, 0.9805161605170362, 0.9663129296065233, 0.9487424991891081, 0.931931984738623, 0.9181974345490227, 0.9075104531838204, 0.8943640941785878, 0.8862664021756573, 0.8757830774574916, 0.86790401991425, 0.8562894022841879, 0.8478155487572495, 0.8387227683262297, 0.8314011142413483, 0.8224071457687988, 0.8161875050523302, 0.8112199678685613, 0.8064334645939688 ], "peak_layer": 1, "peak_normalized": 1.3768889945483427, "population_cv": 0.16325957486062354, "rest_mean": 0.0001431148236776978, "spike_contrast": 0.8564734849544942, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001225740517838858, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0001926341064972803, 0.00018664686649572104, 0.00018061847367789596, 0.00017526224837638438, 0.0001699505082797259, 0.00016466653323732316, 0.00016053428407758474, 0.00015698715287726372, 0.00015311270544771105, 0.00015072032692842185, 0.00014702034241054207, 0.00014439098595175892, 0.00014281776384450495, 0.00013998393842484802, 0.00013717943511437625, 0.00013519232743419707, 0.00013273413060232997, 0.00013038225006312132, 0.00012846071331296116, 0.00012696554767899215, 0.0001251263020094484, 0.00012399339175317436, 0.0001225267187692225, 0.00012142439663875848, 0.00011979944974882528, 0.00011861391249112785, 0.00011734178406186402, 0.00011631744564510882, 0.00011505914153531194, 0.00011418898066040128, 0.00011349399574100971, 0.00011282433843007311 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 0.00014675117381557357, "first_to_last_ratio": 1.2676837465168667, "imbalance_abs_log_ratio": 0.23719141364257146, "last_quartile_mean": 0.00011576323686313117, "mean": 0.00013042020850662084, "normalized": [ 1.169988069421413, 1.156175152925158, 1.1417665260225645, 1.1306726331060184, 1.1183621029407895, 1.1062694829504214, 1.0940115056230622, 1.0844989956600726, 1.0711591662817939, 1.0579765491085218, 1.0443900226505625, 1.035471661013805, 1.0260498632741444, 1.013992390086173, 1.0062479316494695, 0.9977001177294099, 0.9863163473759529, 0.9769959732660659, 0.9687585669786032, 0.9602628595878113, 0.9503845997255937, 0.9426104617668664, 0.9330243108845692, 0.9259760938667744, 0.914797625470175, 0.906289421438531, 0.8956357003805738, 0.888470885240542, 0.8806216558595509, 0.8756991509238615, 0.8712858841028995, 0.8681382926882502 ], "peak_layer": 1, "peak_normalized": 1.169988069421413, "population_cv": 0.09123006364857297, "rest_mean": 0.00013202972385678786, "spike_contrast": 0.9219804606102014, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012172882561571896, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00015259008796419948, 0.00015078860451467335, 0.00014890942838974297, 0.0001474625605624169, 0.00014585701865144074, 0.00014427989663090557, 0.00014268120867200196, 0.00014144058513920754, 0.00013970080181024969, 0.0001379815221298486, 0.00013620956451632082, 0.00013504642993211746, 0.0001338176371064037, 0.00013224509893916547, 0.00013123506505507976, 0.0001301202573813498, 0.00012863558367826045, 0.00012742001854348928, 0.00012634569429792464, 0.00012523768236860633, 0.00012394935765769333, 0.00012293545296415687, 0.00012168522516731173, 0.00012076599523425102, 0.00011930809705518186, 0.00011819845531135798, 0.00011680899478960782, 0.00011587455810513347, 0.00011485085997264832, 0.00011420886585256085, 0.00011363328667357564, 0.00011322277714498341 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00030002388848515693, "first_to_last_ratio": 2.2836575149168885, "imbalance_abs_log_ratio": 0.8257783308640597, "last_quartile_mean": 0.0001313786706305109, "mean": 0.0001947244400071213, "normalized": [ 1.8998833386658438, 1.7835645331795598, 1.6713073472446418, 1.5703972875779748, 1.4780250735507643, 1.3761669788219328, 1.3066447563403325, 1.2401013786965793, 1.1762106459027857, 1.150547637187856, 1.0768735950666006, 1.0264900078881174, 1.0140784159549234, 0.9666871251601796, 0.9096841069789114, 0.8754807117769676, 0.8421876125778792, 0.8073809902734768, 0.77911667677541, 0.7607651063686129, 0.7403822030704592, 0.7308855611321243, 0.7152791523187013, 0.7043381880055872, 0.6944157299089508, 0.686864629891504, 0.680599049276497, 0.6764772714226681, 0.6702412094778135, 0.6672258960044359, 0.6635863560708457, 0.6581114274310631 ], "peak_layer": 1, "peak_normalized": 1.8998833386658438, "population_cv": 0.3592853832280775, "rest_mean": 0.00020492727344390005, "spike_contrast": 0.6813594750078023, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013962913944851608, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0003699537192005664, 0.0003473036049399525, 0.0003254443872720003, 0.00030579473241232336, 0.00028780760476365685, 0.00026797334430739284, 0.0002544356684666127, 0.0002414780465187505, 0.0002290369593538344, 0.00022403974435292184, 0.00020969360775779933, 0.00019988269195891917, 0.0001974658516701311, 0.00018823760910890996, 0.00017713772831484675, 0.00017047749133780599, 0.00016399451124016196, 0.0001572168112033978, 0.00015171305858530104, 0.00014813955931458622, 0.00014417050988413393, 0.00014232128160074353, 0.00013928233238402754, 0.0001371518592350185, 0.00013521971413865685, 0.00013374933041632175, 0.00013252926873974502, 0.00013172665785532445, 0.00013051234418526292, 0.00012992518895771354, 0.00012921648158226162, 0.000128150379168801 ] } } }, "mlp_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "afe29f3ef4d2992353f0bb59fe7c5c2c5050fca671a53184825a8d63cd83a2bc", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.0027073738165199757, 0.002722575329244137, 0.0026850115973502398, 0.0026861224323511124, 0.00271838391199708, 0.0026645169127732515, 0.0026859452482312918, 0.0027359756641089916, 0.0026767742820084095, 0.0027451682835817337, 0.0026999011170119047, 0.002663193503394723, 0.002626648638397455, 0.0026531771291047335, 0.0026517712976783514, 0.002676903735846281, 0.002670086920261383, 0.0026355870068073273, 0.0026610493659973145, 0.0026388936676084995, 0.0026469926815479994, 0.00264862016774714, 0.002641494618728757, 0.002674198243767023, 0.002557571977376938, 0.00252733426168561, 0.0025468147359788418, 0.0026856842450797558, 0.002551089273765683, 0.0025778315030038357, 0.0026001501828432083, 0.0025662274565547705 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.00015858857113926206, "first_to_last_ratio": 3.757867690304466, "imbalance_abs_log_ratio": 1.3238516928952826, "last_quartile_mean": 4.220174423608114e-05, "mean": 8.573221236929385e-05, "normalized": [ 2.5461641896458365, 2.3032502265274117, 2.0924715218207024, 1.8886722264368014, 1.7031663195050915, 1.537434754996766, 1.4223955235407997, 1.3049501874408618, 1.2055297161045848, 1.1692542365736316, 1.0769571962730058, 1.0144505847411098, 0.9913601784485884, 0.9317001386730132, 0.857757082878185, 0.8155634736467603, 0.7682612945963089, 0.7217806418748633, 0.678338533281248, 0.6511820384364709, 0.6192882306865346, 0.6053793146733001, 0.5848721334223365, 0.5718144052517429, 0.5462281883927541, 0.527515083157495, 0.5112993585545178, 0.496920907994256, 0.47722358366421574, 0.46552107689700334, 0.4601234027970099, 0.45317424906679377 ], "peak_layer": 1, "peak_normalized": 2.5461641896458365, "population_cv": 0.5686091737198813, "rest_mean": 9.231269224848668e-05, "spike_contrast": 0.5437781067692293, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.0197621021652594e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00021828828903380781, 0.00019746273756027222, 0.00017939221288543195, 0.0001619200484128669, 0.00014601621660403907, 0.000131807682919316, 0.00012194511509733275, 0.00011187626660102978, 0.00010335272963857278, 0.00010024275252362713, 9.232992306351662e-05, 8.697109296917915e-05, 8.499150135321543e-05, 7.987671415321529e-05, 7.353741239057854e-05, 6.992006092332304e-05, 6.586474046343938e-05, 6.187985127326101e-05, 5.8155463193543255e-05, 5.582727681030519e-05, 5.309295011102222e-05, 5.1900507969548926e-05, 5.0142381951445714e-05, 4.902291402686387e-05, 4.682935104938224e-05, 4.522503513726406e-05, 4.3834825191879645e-05, 4.260212881490588e-05, 4.0913433622336015e-05, 3.991015182691626e-05, 3.944739728467539e-05, 3.885163096128963e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.00021242323236947414, "first_to_last_ratio": 1.8345651540958352, "imbalance_abs_log_ratio": 0.6068074801791611, "last_quartile_mean": 0.0001157894184871111, "mean": 0.00015111673860701558, "normalized": [ 1.771707915810251, 1.6366668487510905, 1.5242909365839221, 1.4186705636795862, 1.3264449439106374, 1.2453857912414694, 1.1893415085703798, 1.1330085309323608, 1.0858463851390703, 1.062861239127963, 1.023237806768965, 0.9942126959777651, 0.978436257513136, 0.9531027445378829, 0.9232414968561398, 0.904013237249231, 0.8853704564278231, 0.8680164030054869, 0.8515402788738432, 0.8395817783841374, 0.8267435189825344, 0.8185143642831039, 0.809172703595369, 0.8007918347797284, 0.7909896391756025, 0.7821052393452608, 0.7753388667989797, 0.7682453290925442, 0.7604995795745633, 0.7549901567959042, 0.7508572350862155, 0.7467737131490535 ], "peak_layer": 1, "peak_normalized": 1.771707915810251, "population_cv": 0.2714170004285523, "rest_mean": 0.00015645500981783562, "spike_contrast": 0.7816309251520459, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012229007406858728, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0002677347220014781, 0.00024732775636948645, 0.00023034587502479553, 0.00021438486874103546, 0.00020044803386554122, 0.0001881986390799284, 0.00017972940986510366, 0.00017121655400842428, 0.00016408956435043365, 0.00016061612404882908, 0.0001546283601783216, 0.00015024218009784818, 0.00014785809617023915, 0.0001440297783119604, 0.00013951724395155907, 0.000136611532070674, 0.0001337942958343774, 0.00013117180787958205, 0.0001286819897359237, 0.00012687486014328897, 0.00012493478425312787, 0.00012369122123345733, 0.00012227953993715346, 0.00012101305037504062, 0.00011953177454415709, 0.00011818919301731512, 0.00011716668086592108, 0.00011609472858253866, 0.00011492421617731452, 0.00011409165017539635, 0.00011346709652571008, 0.00011285000800853595 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.18837904930114746, "first_to_last_ratio": 1.8345651581276443, "imbalance_abs_log_ratio": 0.6068074823768531, "last_quartile_mean": 0.10268321540206671, "mean": 0.13401183625683188, "normalized": [ 1.7717078885231814, 1.6366668119471057, 1.524290957183596, 1.418670514273678, 1.326444920903592, 1.2453858718827995, 1.1893415500975306, 1.1330085716119305, 1.0858463365841577, 1.062861212873729, 1.0232377434953237, 0.9942127340740732, 0.9784362443707608, 0.9531027271661117, 0.9232414917376033, 0.904013241653558, 0.885370477646398, 0.8680164390463408, 0.8515402681847718, 0.8395816969955481, 0.8267435345782925, 0.8185144279720658, 0.8091727254708025, 0.8007918405918669, 0.7909896328038464, 0.7821052665924377, 0.7753388466443417, 0.7682453528580236, 0.7604996016741158, 0.7549901615676585, 0.7508572336421532, 0.7467736753526057 ], "peak_layer": 1, "peak_normalized": 1.7717078885231814, "population_cv": 0.27141699472708714, "rest_mean": 0.1387458680956452, "spike_contrast": 0.7816309473985974, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.10844806432724, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.23742982745170593, 0.2193327248096466, 0.2042730301618576, 0.19011864066123962, 0.1777593195438385, 0.16689644753932953, 0.15938584506511688, 0.151836559176445, 0.1455162614583969, 0.1424359828233719, 0.13712596893310547, 0.13323627412319183, 0.13112203776836395, 0.12772704660892487, 0.1237252876162529, 0.1211484745144844, 0.11865012347698212, 0.11632447689771652, 0.11411647498607635, 0.11251388490200043, 0.11079341918230057, 0.10969062149524689, 0.10843872278928757, 0.10731558501720428, 0.10600197315216064, 0.10481136292219162, 0.10390458256006241, 0.1029539704322815, 0.10191594809293747, 0.10117761790752411, 0.10062375664710999, 0.10007651150226593 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 6.873187476230669e-05, "first_to_last_ratio": 2.248757500145603, "imbalance_abs_log_ratio": 0.8103778415280068, "last_quartile_mean": 3.0564378221242805e-05, "mean": 4.685761763312257e-05, "normalized": [ 1.7400006587426948, 1.6405194725375354, 1.5551306973978087, 1.4944078341288562, 1.4327724019957897, 1.3414187773842097, 1.2804088050749736, 1.2499327782691996, 1.1927955171655855, 1.1608931065609767, 1.0981756972550076, 1.0577423904261343, 1.0542786038595535, 1.0218640087393278, 0.9775807389332813, 0.9570972402588515, 0.9209982861589985, 0.8815092249082567, 0.8478232134430647, 0.8204952153707635, 0.7871780668180614, 0.7759097745574741, 0.7506963524238334, 0.7421155343193898, 0.7072927325438048, 0.6891814936654451, 0.6695399899996352, 0.6616343983211914, 0.6345111343049281, 0.6212687927046295, 0.6200231913288072, 0.6148038704019305 ], "peak_layer": 1, "peak_normalized": 1.7400006587426948, "population_cv": 0.3217536161653951, "rest_mean": 4.900405669845727e-05, "spike_contrast": 0.719671983430414, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.526684668031521e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 8.153228554874659e-05, 7.687083416385576e-05, 7.286971958819777e-05, 7.002439087955281e-05, 6.71363013680093e-05, 6.285568815656006e-05, 5.999690620228648e-05, 5.8568872191244736e-05, 5.5891556257847697e-05, 5.439668530016206e-05, 5.145789691596292e-05, 4.9563288484932855e-05, 4.9400983698433265e-05, 4.7882112994557247e-05, 4.580710447044112e-05, 4.484729652176611e-05, 4.3155785533599555e-05, 4.130542220082134e-05, 3.972697595600039e-05, 3.844645107164979e-05, 3.688528886414133e-05, 3.635728353401646e-05, 3.517584264045581e-05, 3.477376594673842e-05, 3.314205241622403e-05, 3.229340290999971e-05, 3.1373048841487616e-05, 3.10026116494555e-05, 2.9731680115219206e-05, 2.9111175535945222e-05, 2.9052809622953646e-05, 2.8808244678657502e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 0.0001719075116852764, "first_to_last_ratio": 1.4894635014227988, "imbalance_abs_log_ratio": 0.39841598895878805, "last_quartile_mean": 0.00011541572621354135, "mean": 0.00013894635094402474, "normalized": [ 1.3738985779819257, 1.3296082759737424, 1.2883384004275333, 1.2499940655466777, 1.2130730176151365, 1.1755503979222366, 1.147409009170091, 1.11990595257071, 1.0929037170303153, 1.075598370036652, 1.0512762916941902, 1.0312594730951, 1.0194418974669357, 1.0018996501730997, 0.9803460168062212, 0.9648004557293595, 0.9493010807852901, 0.9344788929945884, 0.9198953859218769, 0.908001147597105, 0.8952913731815082, 0.886953676542665, 0.8771894463102521, 0.8683888934041172, 0.8579399877904575, 0.8482799643655919, 0.8408636339472488, 0.8330870307470335, 0.82434015220157, 0.8181187963300758, 0.8135579418994486, 0.809009026741244 ], "peak_layer": 1, "peak_normalized": 1.3738985779819257, "population_cv": 0.16161423472023242, "rest_mean": 0.0001421073115866252, "spike_contrast": 0.8576417505420824, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012187716347398237, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0001908981939777732, 0.0001847442181315273, 0.00017900991952046752, 0.00017368211410939693, 0.00016855206922627985, 0.00016333843814209104, 0.00015942829486448318, 0.00015560684551019222, 0.0001518549834145233, 0.00014945046859793365, 0.00014607100456487387, 0.0001432897406630218, 0.00014164773165248334, 0.00013921030040364712, 0.000136215501697734, 0.0001340555027127266, 0.0001319019211223349, 0.00012984243221580982, 0.0001278161071240902, 0.00012616344611160457, 0.00012439746933523566, 0.00012323897681199014, 0.00012188227265141904, 0.00012065946793882176, 0.00011920763063244522, 0.00011786540562752634, 0.00011683493357850239, 0.00011575440294109285, 0.0001145390560850501, 0.00011367462138878182, 0.00011304090730845928, 0.00011240885214647278 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 0.00014552221000485588, "first_to_last_ratio": 1.262697055715921, "imbalance_abs_log_ratio": 0.2332499537229619, "last_quartile_mean": 0.00011524712863320019, "mean": 0.00012953460168318998, "normalized": [ 1.1676213631634658, 1.1525055688027706, 1.1391580058081985, 1.1289307982409096, 1.1166731577821778, 1.1054001772932092, 1.094813144753513, 1.0822854389705017, 1.0679308603491235, 1.0564894822224282, 1.0456838360716472, 1.0337072702557388, 1.0234945545465333, 1.0158819477934218, 1.0051772952892828, 0.9951467950359763, 0.9866643397779161, 0.9787559418439278, 0.970448736962688, 0.9607908684400951, 0.9509939230158972, 0.9428061355388626, 0.934924980051271, 0.9261034265480621, 0.9170582603044295, 0.9058562786860701, 0.8981778407016658, 0.8907564927732574, 0.8831168120837269, 0.8778216107141017, 0.8745192645108896, 0.8703053916682415 ], "peak_layer": 1, "peak_normalized": 1.1676213631634658, "population_cv": 0.08967236486753827, "rest_mean": 0.00013110875065817877, "spike_contrast": 0.9231588022206524, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012103419721825049, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00015124736819416285, 0.00014928934979252517, 0.000147560378536582, 0.00014623560127802193, 0.00014464781270362437, 0.00014318757166620344, 0.00014181618462316692, 0.00014019341324456036, 0.0001383339986205101, 0.00013685194426216185, 0.00013545223919209093, 0.00013390085950959474, 0.00013257795944809914, 0.00013159186346456409, 0.00013020524056628346, 0.00012890594371128827, 0.00012780717224813998, 0.00012678276107180864, 0.0001257066905964166, 0.0001244556624442339, 0.00012318661902099848, 0.0001221260172314942, 0.00012110513489460573, 0.0001199624384753406, 0.00011879077646881342, 0.00011733973224181682, 0.00011634510883595794, 0.00011538378748809919, 0.0001143941844929941, 0.00011370827269274741, 0.00011328050459269434, 0.0001127346622524783 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00029741139769612346, "first_to_last_ratio": 2.278072415841521, "imbalance_abs_log_ratio": 0.8233296537390045, "last_quartile_mean": 0.00013055396993877366, "mean": 0.00019328556300024502, "normalized": [ 1.889398576482247, 1.780233894901915, 1.6772594716902607, 1.566224958959574, 1.4762232610465842, 1.375571144634086, 1.3083686481730985, 1.2364401527267181, 1.173148897934763, 1.1508864184295666, 1.080457040307468, 1.0318582947152708, 1.0081957004611959, 0.9698994715426978, 0.9066127339785177, 0.8754550048258108, 0.8446146862114313, 0.8115724449181009, 0.7806677558079268, 0.7613480996239201, 0.7403134026279838, 0.7294052762839086, 0.7160186976339193, 0.7062575707090604, 0.6942777320753531, 0.6866005528973866, 0.6829938476675228, 0.6753403086482458, 0.6715967314167978, 0.6686860556999406, 0.6640286432895989, 0.6600445236791291 ], "peak_layer": 1, "peak_normalized": 1.889398576482247, "population_cv": 0.3579154260008552, "rest_mean": 0.00020340604748559632, "spike_contrast": 0.681567477924495, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013863494677934796, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0003651934675872326, 0.0003440935106482357, 0.00032419004128314555, 0.0003027286729775369, 0.0002853326441254467, 0.00026587804313749075, 0.00025288877077400684, 0.00023898603103589267, 0.00022675274522043765, 0.00022244972933549434, 0.0002088367473334074, 0.00019944331143051386, 0.00019486967357806861, 0.00018746756541077048, 0.00017523515271022916, 0.00016921281348913908, 0.0001632518251426518, 0.00015686523693148047, 0.00015089180669747293, 0.00014715759607497603, 0.00014309189282357693, 0.00014098350948188454, 0.00013839607709087431, 0.0001365093921776861, 0.0001341938623227179, 0.00013270997442305088, 0.00013201285037212074, 0.00013053353177383542, 0.0001298099523410201, 0.00012924736074637622, 0.00012834715016651899, 0.00012757707736454904 ] } } }, "post_attention_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "31853ffb07b24cc59e774bb20656840b4358b3080b42516d380f2dc0f46404c8", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.0022265110164880753, 0.0040087271481752396, 0.005379301495850086, 0.006410556845366955, 0.0027253651060163975, 0.004838948138058186, 0.006175771821290255, 0.00742313964292407, 0.0030937574338167906, 0.005150026176124811, 0.007019025273621082, 0.008143599145114422, 0.004255065228790045, 0.0066224560141563416, 0.008129785768687725, 0.010099578648805618, 0.0041999719105660915, 0.00719039561226964, 0.009030314162373543, 0.010736005380749702, 0.005131533369421959, 0.00787449348717928, 0.009912566281855106, 0.012113703414797783, 0.0059690922498703, 0.00876442901790142, 0.010806876234710217, 0.012353328987956047, 0.006502739619463682, 0.009470963850617409, 0.012123330496251583, 0.01384514570236206 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.00015962236011546338, "first_to_last_ratio": 3.7698422272252903, "imbalance_abs_log_ratio": 1.3270331510418396, "last_quartile_mean": 4.2341920561739244e-05, "mean": 8.61727822893954e-05, "normalized": [ 2.542734798705971, 2.3221448837042376, 2.101145461274622, 1.8911858524935854, 1.7004204056817789, 1.5354142426015858, 1.4229001770142031, 1.3028733810328659, 1.2049412416934622, 1.1705258152816296, 1.0773908232649334, 1.0133220774376726, 0.990609360100461, 0.9272534949358001, 0.8550385909364457, 0.8185651575045715, 0.7703972051466339, 0.7173947009730917, 0.6760256695667757, 0.6514523109028573, 0.618419077603279, 0.6053620388084041, 0.5838930841859182, 0.5697043695032278, 0.5457802700901755, 0.5276149005446632, 0.5104701332743111, 0.49517988582787786, 0.47527562953524777, 0.46579059316028215, 0.45896435764104826, 0.45181000957238127 ], "peak_layer": 1, "peak_normalized": 2.542734798705971, "population_cv": 0.5705202801130907, "rest_mean": 9.280119640472414e-05, "spike_contrast": 0.542873885449774, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.03793460666202e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00021911453222855926, 0.00020010568550787866, 0.00018106155039276928, 0.00016296874673571438, 0.00014652995741926134, 0.00013231091725174338, 0.0001226152671733871, 0.00011227222421439365, 0.00010383313929196447, 0.00010086746624438092, 9.284176485380158e-05, 8.732078276807442e-05, 8.536356472177431e-05, 7.99040135461837e-05, 7.368105434579775e-05, 7.053803710732609e-05, 6.638727063545957e-05, 6.181989738252014e-05, 5.825501284562051e-05, 5.613745815935545e-05, 5.329089253791608e-05, 5.216573117650114e-05, 5.0315691623836756e-05, 4.9093010602518916e-05, 4.703140439232811e-05, 4.546604395727627e-05, 4.398863165988587e-05, 4.267102849553339e-05, 4.095582335139625e-05, 4.013847137684934e-05, 3.955023566959426e-05, 3.893372559105046e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.00021442477373057045, "first_to_last_ratio": 1.8431254562763404, "imbalance_abs_log_ratio": 0.6114627481432037, "last_quartile_mean": 0.00011633759004325839, "mean": 0.00015221248349917005, "normalized": [ 1.7740805953138137, 1.6484457246979003, 1.5301434809496548, 1.4217241600875643, 1.3264958046529274, 1.2454577490392524, 1.1897861330408546, 1.1336265611690741, 1.086914920172498, 1.064301358569379, 1.0227209980571423, 0.9941753882589759, 0.9788979908211966, 0.9511107967643203, 0.9228143274453928, 0.9052447611165039, 0.8846530997418666, 0.8652930871753776, 0.8497381552743479, 0.8388656494639406, 0.8256334792987466, 0.8176309647342147, 0.8076951735882674, 0.8000661307359944, 0.7892605467734257, 0.78144613040264, 0.7731219607566042, 0.7664988488799842, 0.7584935618388989, 0.7529639051546407, 0.7485195296370439, 0.7441790263875565 ], "peak_layer": 1, "peak_normalized": 1.7740805953138137, "population_cv": 0.2736411578809574, "rest_mean": 0.00015762286892384981, "spike_contrast": 0.7803207938393821, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012299640220589936, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00027003721334040165, 0.00025091401766985655, 0.00023290693934541196, 0.0002164041652576998, 0.000201909220777452, 0.0001895742170745507, 0.00018110030214302242, 0.00017255211423616856, 0.00016544201935175806, 0.00016199995297938585, 0.00015567090304102749, 0.00015132590488065034, 0.0001490004942752421, 0.00014477093645837158, 0.00014046386058907956, 0.00013778955326415598, 0.0001346552453469485, 0.00013170840975362808, 0.00012934075493831187, 0.00012768582382705063, 0.00012567172234412283, 0.00012445363972801715, 0.00012294128828216344, 0.00012178005272289738, 0.00012013530795229599, 0.00011894585622940212, 0.00011767881369451061, 0.00011667069338727742, 0.00011545218876563013, 0.00011461050598882139, 0.00011393401655368507, 0.00011327333777444437 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.190154030919075, "first_to_last_ratio": 1.84312545379361, "imbalance_abs_log_ratio": 0.6114627467961817, "last_quartile_mean": 0.10316933691501617, "mean": 0.13498355145566165, "normalized": [ 1.774080628234443, 1.6484456620184311, 1.5301434020504485, 1.4217242444591272, 1.326495766996536, 1.2454576834375484, 1.189786122599316, 1.133626627255238, 1.0869149569673349, 1.0643014004698326, 1.0227209907477024, 0.9941753870542451, 0.9788979589698585, 0.9511107512164827, 0.9228143032197887, 0.9052447972745712, 0.8846531331847195, 0.8652931173486506, 0.8497382179112226, 0.8388656682549475, 0.8256334805754935, 0.8176309131099365, 0.8076951535479161, 0.8000661540388709, 0.7892605573866218, 0.7814461538045975, 0.7731219580171443, 0.7664988542586116, 0.7584935821787124, 0.7529638601652359, 0.7485195164462104, 0.7441789968002049 ], "peak_layer": 1, "peak_normalized": 1.774080628234443, "population_cv": 0.27364115345546325, "rest_mean": 0.13978153549962574, "spike_contrast": 0.7803207857775153, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.10907443761825561, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.2394717037677765, 0.22251304984092712, 0.2065441906452179, 0.19190938770771027, 0.1790551096200943, 0.16811630129814148, 0.16060155630111694, 0.1530209481716156, 0.14671564102172852, 0.1436631828546524, 0.13805051147937775, 0.13419732451438904, 0.13213512301445007, 0.1283843070268631, 0.1245647519826889, 0.12219315767288208, 0.11941362172365189, 0.11680033802986145, 0.11470068246126175, 0.1132330670952797, 0.11144693940877914, 0.11036672443151474, 0.10902556031942368, 0.10799577087163925, 0.10653719305992126, 0.10548237711191177, 0.1043587476015091, 0.10346473753452301, 0.10238415747880936, 0.10163773596286774, 0.10103782266378403, 0.10045192390680313 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 6.883367768750759e-05, "first_to_last_ratio": 2.249759357551583, "imbalance_abs_log_ratio": 0.810823258297238, "last_quartile_mean": 3.059601795030176e-05, "mean": 4.691846504556452e-05, "normalized": [ 1.7359059710084281, 1.6475789727938472, 1.5536456441204052, 1.4921194549590373, 1.4365657321885223, 1.3438513058553838, 1.2787343947807137, 1.2483299149152907, 1.1912354282513808, 1.1603083401306369, 1.099547144002948, 1.059972440022979, 1.0583955432739494, 1.0176800955770247, 0.9754963047746087, 0.9631595708780456, 0.9242400656751482, 0.8752301116137658, 0.8454245377074081, 0.8232853322610043, 0.7863494898171388, 0.7774642190168813, 0.7497486129019063, 0.7388483538626245, 0.709315327139865, 0.6908343794807076, 0.6685469983108738, 0.6592365073812129, 0.6314341746588822, 0.6235898169926835, 0.6199474545603838, 0.6139783610863121 ], "peak_layer": 1, "peak_normalized": 1.7359059710084281, "population_cv": 0.3220406003070692, "rest_mean": 4.9070239684725595e-05, "spike_contrast": 0.7193541792517962, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.52988819940947e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 8.144604362314567e-05, 7.730187644483522e-05, 7.28946688468568e-05, 7.000795449130237e-05, 6.740145909134299e-05, 6.305144052021205e-05, 5.999625500408001e-05, 5.8569723478285596e-05, 5.589093780145049e-05, 5.4439886298496276e-05, 5.158906424185261e-05, 4.973227987647988e-05, 4.9658294301480055e-05, 4.774798799189739e-05, 4.576878927764483e-05, 4.51899686595425e-05, 4.3363925215089694e-05, 4.1064453398576006e-05, 3.9666021621087566e-05, 3.86272840842139e-05, 3.689431105158292e-05, 3.6477427784120664e-05, 3.5177054087398574e-05, 3.466563066467643e-05, 3.3279986382694915e-05, 3.2412888685939834e-05, 3.136719897156581e-05, 3.093036502832547e-05, 2.9625922252307646e-05, 2.9257877031341195e-05, 2.9086982976878062e-05, 2.8806922273361124e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 0.00017341253987979144, "first_to_last_ratio": 1.4955148931299893, "imbalance_abs_log_ratio": 0.4024705576674927, "last_quartile_mean": 0.00011595507385209203, "mean": 0.00013990553861731314, "normalized": [ 1.3768818258178377, 1.3340879911950096, 1.2910054180628048, 1.2527236427111486, 1.2147542166907819, 1.1769889670731195, 1.1474411155803916, 1.1220953663134672, 1.0944075691302455, 1.0773007485615413, 1.0508678613623372, 1.0320596054419082, 1.0208161736522385, 1.000559648637946, 0.9805197815181176, 0.9663128275349431, 0.948744191771126, 0.9319305822463463, 0.9182019814348666, 0.9075038868226692, 0.8943650364968407, 0.886262676120959, 0.8757907045344768, 0.8679002174812178, 0.856288373659771, 0.8478178092769747, 0.8387215581135022, 0.8313975747674917, 0.8224084044020752, 0.8161864847722404, 0.8112200992010112, 0.8064376596145924 ], "peak_layer": 1, "peak_normalized": 1.3768818258178377, "population_cv": 0.16325888721442564, "rest_mean": 0.0001431150389921472, "spike_contrast": 0.8564734877369171, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001225742365932092, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0001926333934534341, 0.00018664629897102714, 0.00018061880837194622, 0.0001752629759721458, 0.00016995084297377616, 0.0001646672753849998, 0.00016053336730692536, 0.00015698735660407692, 0.00015311368042603135, 0.00015072034148033708, 0.00014702223415952176, 0.00014439085498452187, 0.0001428178366040811, 0.00013998383656144142, 0.00013718014815822244, 0.00013519251660909504, 0.00013273456715978682, 0.00013038225006312132, 0.00012846154277212918, 0.00012696482008323073, 0.00012512662215158343, 0.0001239930570591241, 0.00012252797023393214, 0.000121424047392793, 0.00011979948612861335, 0.00011861440725624561, 0.00011734179133782163, 0.0001163171255029738, 0.00011505949078127742, 0.00011418900976423174, 0.00011349418491590768, 0.00011282509512966499 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 0.00014675183047074825, "first_to_last_ratio": 1.2676545313577883, "imbalance_abs_log_ratio": 0.23716836728277432, "last_quartile_mean": 0.00011576642282307148, "mean": 0.00013041823922321782, "normalized": [ 1.1700698937843759, 1.1560440994564782, 1.1417398043557874, 1.130463200949706, 1.118403537277814, 1.1065304334184665, 1.0939881912841023, 1.0846815122144253, 1.0709468270677047, 1.0579907390333845, 1.0442838370166418, 1.035550673160169, 1.025959021745254, 1.0140203095307772, 1.0062047700154308, 0.9976631870190885, 0.9861765922450596, 0.9770016877798172, 0.9687702939454803, 0.9602422119882037, 0.9504995944091904, 0.9425883202531964, 0.9330014667281973, 0.9259385264566372, 0.9147443798098084, 0.906202796813466, 0.8958065504175365, 0.8885655303249116, 0.8806194435723184, 0.8757905905296257, 0.8713783728577705, 0.8681336045391749 ], "peak_layer": 1, "peak_normalized": 1.1700698937843759, "population_cv": 0.09121429133712369, "rest_mean": 0.0001320278307604086, "spike_contrast": 0.921975648780332, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001217264449223876, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0001525984553154558, 0.0001507692359155044, 0.000148903694935143, 0.00014743302017450333, 0.00014586022007279098, 0.00014431175077334046, 0.00014267601363826543, 0.00014146225294098258, 0.000139670999487862, 0.00013798128929920495, 0.00013619365927297622, 0.00013505469541996717, 0.00013380376913119107, 0.00013224674330558628, 0.00013122745440341532, 0.00013011347618885338, 0.00012861541472375393, 0.00012741883983835578, 0.0001263453159481287, 0.0001252330985153094, 0.0001239624834852293, 0.00012293070903979242, 0.00012168040848337114, 0.00012075927224941552, 0.00011929935135412961, 0.00011818537313956767, 0.00011682951299007982, 0.00011588515189941972, 0.00011484883725643158, 0.00011421906674513593, 0.00011364363308530301, 0.00011322045611450449 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00030004180916876066, "first_to_last_ratio": 2.2837506327398005, "imbalance_abs_log_ratio": 0.8258191057718037, "last_quartile_mean": 0.00013138116082700435, "mean": 0.00019473525253488333, "normalized": [ 1.9000096511818072, 1.783700741635724, 1.6708066910504589, 1.570366884717585, 1.4783058799917985, 1.376185321596543, 1.3064973297630273, 1.2402700043336092, 1.1767953849206394, 1.1505341943958838, 1.0771542571420398, 1.026098162637436, 1.013898063759585, 0.9669473025647412, 0.9095398902231254, 0.8755205777387188, 0.842301214161826, 0.8073124727024382, 0.7791991818813049, 0.760574607765185, 0.7402055397447304, 0.7308143413325056, 0.7152667122155245, 0.7043714155678553, 0.6942467749765324, 0.6868749151283189, 0.680655564574882, 0.676363713518914, 0.6701061029149729, 0.6672276319278244, 0.6636177363814285, 0.6582317375530352 ], "peak_layer": 1, "peak_normalized": 1.9000096511818072, "population_cv": 0.35930746026536914, "rest_mean": 0.00020494150898971216, "spike_contrast": 0.6812747128050888, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013962146767880766, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0003699988592416048, 0.0003473494143690914, 0.0003253649629186839, 0.0003058057918678969, 0.0002878782688640058, 0.0002679917961359024, 0.0002544210874475539, 0.00024152429250534624, 0.00022916354646440595, 0.000224049566895701, 0.00020975990628357977, 0.00019981748482678086, 0.000197441695490852, 0.0001882987271528691, 0.00017711948021315038, 0.00017049472080543637, 0.00016402573965024203, 0.00015721219824627042, 0.00015173754945863038, 0.00014811068831477314, 0.00014414411270990968, 0.0001423153153154999, 0.0001392876438330859, 0.00013716594548895955, 0.00013519432104658335, 0.00013375876005738974, 0.00013254763325676322, 0.0001317118585575372, 0.00013049328117631376, 0.00012993274140171707, 0.00012922976748086512, 0.00012818092363886535 ] } } }, "post_mlp_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "7fd9c911e93175896ad618842c120e77b3f017c17b7eec9c2bfb01fc6dbc8ac2", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.0034885562490671873, 0.004831995815038681, 0.005952076520770788, 0.006925336085259914, 0.00385080324485898, 0.005542940925806761, 0.006729288958013058, 0.007927131839096546, 0.004081308841705322, 0.005811207927763462, 0.007534808944910765, 0.008581922389566898, 0.00499920267611742, 0.007102519739419222, 0.008542952127754688, 0.010447698645293713, 0.005041283555328846, 0.007636935915797949, 0.009387043304741383, 0.011020516976714134, 0.005807018838822842, 0.008367395959794521, 0.01024632342159748, 0.012378672137856483, 0.006453148555010557, 0.009127611294388771, 0.011037957854568958, 0.012611051090061665, 0.007010402157902718, 0.009794782847166061, 0.012445803731679916, 0.014087481424212456 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.00015858902861509705, "first_to_last_ratio": 3.757869257587208, "imbalance_abs_log_ratio": 1.3238521099622325, "last_quartile_mean": 4.220184837322449e-05, "mean": 8.57325238712292e-05, "normalized": [ 2.5462257183789148, 2.3032637538343064, 2.0924289533379263, 1.8886492391629932, 1.7031256747388024, 1.5374176267930122, 1.4224231993460965, 1.3049597038569594, 1.2055618292025094, 1.169245405308266, 1.0769812048488787, 1.0144334047898542, 0.9913512297333369, 0.9317090592952362, 0.8577499774884012, 0.81555804918997, 0.7682725064221458, 0.7217807351250844, 0.6783324192661732, 0.6511800118936032, 0.6192916242842709, 0.6053728292385717, 0.5848720876072855, 0.5718124973482859, 0.5462302773879167, 0.5275173250132701, 0.5112998770983732, 0.49691833866078555, 0.47721658788927634, 0.46551989467389665, 0.46012635628943105, 0.45317260249616614 ], "peak_layer": 1, "peak_normalized": 2.5462257183789148, "population_cv": 0.5686111305901927, "rest_mean": 9.231303704803361e-05, "spike_contrast": 0.5437775023084314, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.019775271648541e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00021829435718245804, 0.00019746461475733668, 0.0001793892151908949, 0.0001619186659809202, 0.00014601326256524771, 0.00013180669338908046, 0.00012194793089292943, 0.00011187748896190897, 0.00010335585830034688, 0.00010024235962191597, 9.233231685357168e-05, 8.69699360919185e-05, 8.499104296788573e-05, 7.987776916706935e-05, 7.353707042057067e-05, 6.991984992055222e-05, 6.586594099644572e-05, 6.188008410390466e-05, 5.8155150327365845e-05, 5.582730591413565e-05, 5.309343396220356e-05, 5.190014053368941e-05, 5.014256021240726e-05, 4.90229285787791e-05, 4.682970029534772e-05, 4.522539165918715e-05, 4.383502891869284e-05, 4.2602063331287354e-05, 4.0912982512963936e-05, 3.991019548266195e-05, 3.944779382436536e-05, 3.885163096128963e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.00021242368711682502, "first_to_last_ratio": 1.8345658968465297, "imbalance_abs_log_ratio": 0.6068078850437921, "last_quartile_mean": 0.0001157896194854402, "mean": 0.0001511171253696375, "normalized": [ 1.7717451736709768, 1.636674793183293, 1.5242755762025908, 1.4186515254985965, 1.3264266232459345, 1.2453738409548603, 1.1893512719369177, 1.1330135273991528, 1.085860457802012, 1.0628542818802846, 1.023252039669589, 0.9942070699734702, 0.9784308644775869, 0.9531083940389613, 0.9232390376562555, 0.9040072643238835, 0.8853775311228679, 0.8680181295617843, 0.8515367513400655, 0.8395795332993111, 0.8267439067367663, 0.8185133286645813, 0.8091730400239778, 0.800791325997512, 0.7909899739933552, 0.7821072339266707, 0.7753403490721149, 0.7682422073314642, 0.7604971517068638, 0.7549863467424895, 0.7508596466770359, 0.7467718018887739 ], "peak_layer": 1, "peak_normalized": 1.7717451736709768, "population_cv": 0.2714186609524326, "rest_mean": 0.00015645541295978345, "spike_contrast": 0.7816308178118705, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012229037238284946, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0002677410375326872, 0.0002473295899108052, 0.00023034414334688336, 0.00021438254043459892, 0.00020044577831868082, 0.00018819731485564262, 0.00017973134526982903, 0.000171217747265473, 0.0001640921109355986, 0.00016061548376455903, 0.00015463090676348656, 0.00015024171443656087, 0.0001478576596127823, 0.00014403100067283958, 0.00013951722939964384, 0.00013661097909789532, 0.00013379570737015456, 0.0001311724045081064, 0.0001286817860091105, 0.00012687484559137374, 0.0001249351626029238, 0.00012369138130452484, 0.00012227990373503417, 0.00012101328320568427, 0.00011953213106608018, 0.0001181897969217971, 0.0001171672047348693, 0.00011609455395955592, 0.00011492414341773838, 0.0001140913664130494, 0.00011346775136189535, 0.00011285000800853595 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.18837944976985455, "first_to_last_ratio": 1.8345658800697515, "imbalance_abs_log_ratio": 0.6068078758989703, "last_quartile_mean": 0.10268339328467846, "mean": 0.1340121782850474, "normalized": [ 1.7717451751532867, 1.6366747548098712, 1.5242756140248972, 1.4186514377480737, 1.3264266357207033, 1.2453739091726768, 1.1893513017871604, 1.1330134634090196, 1.0858604665419038, 1.0628542749024097, 1.0232520332438115, 0.9942070832344971, 0.9784308561813967, 0.9531084116998004, 0.9232390798282929, 0.9040072094634526, 0.8853775581660035, 0.8680181154203164, 0.8515367049614931, 0.8395794986042229, 0.8267439263823189, 0.8185133396785964, 0.809173050926764, 0.8007913534932437, 0.7909900046670136, 0.782107217826699, 0.7753403703776178, 0.7682422246066585, 0.7604971603471317, 0.7549863443945886, 0.7508596538022533, 0.7467717694238248 ], "peak_layer": 1, "peak_normalized": 1.7717451751532867, "population_cv": 0.27141865758333467, "rest_mean": 0.13874622461972413, "spike_contrast": 0.7816308398662988, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.10844832807779312, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.23743543028831482, 0.21933434903621674, 0.20427149534225464, 0.19011656939983368, 0.17775732278823853, 0.16689527034759521, 0.15938755869865417, 0.15183760225772858, 0.14551852643489838, 0.14243541657924652, 0.13712823390960693, 0.1332358568906784, 0.1311216503381729, 0.12772813439369202, 0.1237252801656723, 0.12114797532558441, 0.1186513751745224, 0.1163249984383583, 0.11411628872156143, 0.11251387745141983, 0.11079375445842743, 0.10969075560569763, 0.10843904316425323, 0.107315793633461, 0.10600229352712631, 0.104811891913414, 0.10390505194664001, 0.10295381397008896, 0.1019158810377121, 0.10117736458778381, 0.10062433779239655, 0.10007651150226593 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 6.873207303215167e-05, "first_to_last_ratio": 2.248757814140914, "imbalance_abs_log_ratio": 0.8103779811585758, "last_quartile_mean": 3.0564462122129044e-05, "mean": 4.6857780716891284e-05, "normalized": [ 1.7400119939228862, 1.6405409364325143, 1.555121247708426, 1.4944054279879166, 1.4327475398601994, 1.3414100714975594, 1.2804101716466747, 1.2499370459040815, 1.192804641985012, 1.1608858830017554, 1.098191983597808, 1.057744066133373, 1.0542717513613309, 1.021869458338179, 0.9775749297615711, 0.9570927445976433, 0.9210091333285576, 0.8815125232807631, 0.847820728512489, 0.8204970180441978, 0.7871838673851995, 0.7759041238100245, 0.7506938949785397, 0.7421070509174572, 0.7072987335050381, 0.6891811912847667, 0.6695407652827761, 0.6616333377893114, 0.6345027924930273, 0.6212629426015038, 0.6200302724102036, 0.614801730639213 ], "peak_layer": 1, "peak_normalized": 1.7400119939228862, "population_cv": 0.32175451551289735, "rest_mean": 4.9004235565748616e-05, "spike_contrast": 0.7196709452950106, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.5266924533061685e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 8.153310045599937e-05, 7.687210745643824e-05, 7.28695304132998e-05, 7.002452184678987e-05, 6.713537004543468e-05, 6.28554989816621e-05, 5.9997179050697014e-05, 5.8569276006892323e-05, 5.589217835222371e-05, 5.439653614303097e-05, 5.1458839152473956e-05, 4.9563539505470544e-05, 4.9400834541302174e-05, 4.788253500009887e-05, 4.58069916930981e-05, 4.4847241952084005e-05, 4.315644400776364e-05, 4.130572051508352e-05, 3.972699778387323e-05, 3.8446669350378215e-05, 3.688568904181011e-05, 3.635714529082179e-05, 3.5175849916413426e-05, 3.477348946034908e-05, 3.3142448955914006e-05, 3.2293501135427505e-05, 3.13731943606399e-05, 3.1002669857116416e-05, 2.9731392714893445e-05, 2.9111002731951885e-05, 2.905324254243169e-05, 2.8808244678657502e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 0.00017190765356644988, "first_to_last_ratio": 1.4894621368061003, "imbalance_abs_log_ratio": 0.3984150727783497, "last_quartile_mean": 0.00011541592721187044, "mean": 0.00013894661265112518, "normalized": [ 1.3739040544627474, 1.3296102750467884, 1.2883342981472405, 1.2499918159027226, 1.2130627732820631, 1.175544413473833, 1.1474083142393565, 1.1199112790671786, 1.092909618037032, 1.0755929927697745, 1.0512861461262244, 1.0312571117882559, 1.0194388253067554, 1.0019061415077835, 0.980344379776871, 0.9647961249933756, 0.9493075664595542, 0.9344817410273759, 0.9198929201794295, 0.9079992279088673, 0.8952920956888009, 0.8869530532606651, 0.8771902552778059, 0.8683889858357474, 0.8579407806493952, 0.8482823463732029, 0.8408658204630546, 0.8330843095877946, 0.8243382853592501, 0.818115422616924, 0.8135611224181157, 0.8090075029660142 ], "peak_layer": 1, "peak_normalized": 1.3739040544627474, "population_cv": 0.16161403917150835, "rest_mean": 0.00014210757001786045, "spike_contrast": 0.8576421569620596, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012187744287075474, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00019089931447524577, 0.00018474484386388212, 0.00017900968668982387, 0.00017368212866131216, 0.0001685509632807225, 0.00016333791427314281, 0.00015942849859129637, 0.00015560787869617343, 0.00015185608936008066, 0.00014945000293664634, 0.00014607264893129468, 0.0001432896824553609, 0.00014164757158141583, 0.0001392114645568654, 0.00013621553080156446, 0.0001340551534667611, 0.00013190307072363794, 0.00012984307250007987, 0.0001278160052606836, 0.00012616341700777411, 0.0001243978040292859, 0.00012323912233114243, 0.00012188261462142691, 0.00012065970804542303, 0.00011920796532649547, 0.00011786595860030502, 0.00011683545744745061, 0.00011575424287002534, 0.00011453901242930442, 0.00011367436673026532, 0.00011304156214464456, 0.00011240885214647278 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 0.00014552358334185556, "first_to_last_ratio": 1.262638843036328, "imbalance_abs_log_ratio": 0.2332038508028458, "last_quartile_mean": 0.00011525352965691127, "mean": 0.00012953498708156985, "normalized": [ 1.1676304712442283, 1.1525836984023852, 1.1391521450612143, 1.1288261090295841, 1.1167600441451297, 1.1054423860149858, 1.094731137320412, 1.082319740349084, 1.067957453000188, 1.0563935463552665, 1.04560972624621, 1.0337959762176818, 1.0235535208863231, 1.0159040893770892, 1.0050628637064216, 0.9950792389307415, 0.9865814183778941, 0.9786230528286792, 0.9704418054120744, 0.9608381133357455, 0.9508407831226554, 0.9429749854457838, 0.9346839821902475, 0.9262276149685831, 0.9170542960948607, 0.9059488475652709, 0.8984648923618161, 0.8908037775288885, 0.8831483920181569, 0.8778579808382188, 0.8744051093280881, 0.8703028022960919 ], "peak_layer": 1, "peak_normalized": 1.1676304712442283, "population_cv": 0.0896642712448568, "rest_mean": 0.00013110964479252557, "spike_contrast": 0.9231344927669946, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012103183544240892, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00015124899800866842, 0.00014929991448298097, 0.00014756005839444697, 0.00014622247545048594, 0.00014465949789155275, 0.00014319346519187093, 0.00014180598373059183, 0.00014019827358424664, 0.00013833785487804562, 0.00013683992438018322, 0.0001354430423816666, 0.00013391274842433631, 0.0001325859921053052, 0.00013159512309357524, 0.00013019080506637692, 0.00012889757636003196, 0.00012779681128449738, 0.00012676592450588942, 0.00012570616672746837, 0.00012446215259842575, 0.00012316714855842292, 0.00012214825255796313, 0.00012107427755836397, 0.00011997888213954866, 0.00011879061639774591, 0.00011735207226593047, 0.0001163826382253319, 0.0001153902558144182, 0.00011439861555118114, 0.00011371332220733166, 0.00011326605454087257, 0.0001127346622524783 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00029742828155576717, "first_to_last_ratio": 2.2781890121807993, "imbalance_abs_log_ratio": 0.8233808344454435, "last_quartile_mean": 0.00013055469935352448, "mean": 0.00019329176166138495, "normalized": [ 1.8891850069388627, 1.7801603926271052, 1.6773544463608039, 1.566500262900107, 1.476456430986117, 1.3754925511314788, 1.3084504582544147, 1.236424592515421, 1.172965600373877, 1.150871569148191, 1.0802294364746856, 1.0319209662753672, 1.0080063247818516, 0.9698985570693867, 0.9067931019504507, 0.8756163462587222, 0.8444435806568995, 0.8113939671049883, 0.7809677247391495, 0.7613442367276676, 0.7402582678385915, 0.7293562882295931, 0.7162426695419845, 0.7062419232587185, 0.6940340549797547, 0.6866100033591334, 0.6831688143117768, 0.6753320518823956, 0.6715919825156708, 0.6686410475337363, 0.6640239865195754, 0.6600233567535221 ], "peak_layer": 1, "peak_normalized": 1.8891850069388627, "population_cv": 0.357915029863326, "rest_mean": 0.00020341356920888993, "spike_contrast": 0.6815376252628051, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001386340009048581, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00036516389809548855, 0.0003440903383307159, 0.0003242187958676368, 0.0003027915954589844, 0.00028538686456158757, 0.00026587137836031616, 0.0002529126941226423, 0.00023899068764876574, 0.00022672458726447076, 0.00022245399304665625, 0.0002087994507746771, 0.00019946182146668434, 0.00019483931828290224, 0.00018747340072877705, 0.00017527563613839447, 0.00016924942610785365, 0.00016322398732881993, 0.00015683576930314302, 0.0001509546273155138, 0.00014716156874783337, 0.0001430858246749267, 0.0001409785618307069, 0.00013844380737282336, 0.00013651074550580233, 0.00013415106514003128, 0.00013271605712361634, 0.00013205090363044292, 0.00013053612201474607, 0.00012981319741811603, 0.00012924280599690974, 0.00012835036613978446, 0.00012757707736454904 ] } } }, "pre_attention_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "de71d4cf2d509a1063f6eac7f50c4885e9a058dee7e4b993ad40f5527f528d00", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.02802751213312149, 0.01413465291261673, 0.014225807972252369, 0.01433455292135477, 0.01440478302538395, 0.009703479707241058, 0.00977881159633398, 0.009858808480203152, 0.009950309991836548, 0.00752287870272994, 0.007571155205368996, 0.007693085819482803, 0.007788818795233965, 0.006353048607707024, 0.006389216054230928, 0.0064707002602517605, 0.00656962301582098, 0.005553483963012695, 0.0056249531917274, 0.005688921548426151, 0.005768724251538515, 0.004966245498508215, 0.005057469941675663, 0.005098348017781973, 0.005216081161051989, 0.0046758754178881645, 0.004727107938379049, 0.004777040798217058, 0.004837613552808762, 0.0044186655431985855, 0.004510571248829365, 0.0045849340967834 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.0001580166108396952, "first_to_last_ratio": 1.9713740353643698, "imbalance_abs_log_ratio": 0.6787307795042399, "last_quartile_mean": 8.015557068574708e-05, "mean": 0.00011843875881822896, "normalized": [ 0.8918679597819287, 1.5814819593588525, 1.5722239987443252, 1.3246067564285042, 1.1252650527356378, 1.512743589128775, 1.3788112428745087, 1.2863035124436846, 1.097095055843533, 1.468154353173571, 1.1523254606068964, 1.0839888571909224, 1.054518217028451, 1.146335080549274, 1.1665208527784754, 0.8931136212023528, 0.8203804970547371, 1.073741223277434, 0.919654742394771, 0.8223971330190447, 0.7711288002244273, 0.8470775332584999, 0.8298711629738247, 0.7662487463744663, 0.7210382462310316, 0.7071376059209309, 0.7189559392079661, 0.727980518763353, 0.6111916361855597, 0.6986829946379066, 0.6244678178312321, 0.6046898327751223 ], "peak_layer": 2, "peak_normalized": 1.5814819593588525, "population_cv": 0.2904715201464359, "rest_mean": 0.00012310891098523926, "spike_contrast": 0.7572151875143335, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.321993711637333e-05, "top_five_layers": [ 2, 3, 6, 10, 7 ], "top_five_spike_overlap": 0 }, "values": [ 0.00010563173418631777, 0.0001873087603598833, 0.00018621225899551064, 0.00015688478015363216, 0.0001332749961875379, 0.00017916747310664505, 0.00016330469225067645, 0.00015234819147735834, 0.00012993857671972364, 0.00017388637934345752, 0.0001364799973089248, 0.0001283862948184833, 0.00012489582877606153, 0.00013577050413005054, 0.00013816128193866462, 0.00010577926877886057, 9.71648478298448e-05, 0.00012717257777694613, 0.00010892276623053476, 9.74036956904456e-05, 9.133153798757121e-05, 0.0001003268116619438, 9.828891052166e-05, 9.07535504666157e-05, 8.539887494407594e-05, 8.375250035896897e-05, 8.515224908478558e-05, 8.622110908618197e-05, 7.238877878990024e-05, 8.275114669231698e-05, 7.396119326585904e-05, 7.161871326388791e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.0001664285200604354, "first_to_last_ratio": 1.9719337364651075, "imbalance_abs_log_ratio": 0.6790146534176558, "last_quartile_mean": 8.439863722742302e-05, "mean": 0.0001245867656507471, "normalized": [ 0.8984334461398115, 1.5927952483854981, 1.5658595485525297, 1.3198735865176374, 1.131213894321221, 1.5125315092450449, 1.3796810696180641, 1.286366052942558, 1.0960606282761096, 1.4536884557404346, 1.1543342746982923, 1.0797559630746076, 1.0497530570909623, 1.1457182987306291, 1.1551962694199982, 0.9023721663305979, 0.8238710779144349, 1.0632503993742801, 0.9211624832492573, 0.8274335806257342, 0.7739032412019898, 0.850990856837025, 0.8281317613098801, 0.7681943949939335, 0.7208251570035091, 0.7089546837555952, 0.7203162530736348, 0.7236174125342815, 0.6145740290254166, 0.6994991973458914, 0.6274381906330746, 0.6042038120380651 ], "peak_layer": 2, "peak_normalized": 1.5927952483854981, "population_cv": 0.2889665774838781, "rest_mean": 0.00012946851159169995, "spike_contrast": 0.7586812913967177, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.822533756960183e-05, "top_five_layers": [ 2, 3, 6, 10, 7 ], "top_five_spike_overlap": 0 }, "values": [ 0.00011193291720701382, 0.0001984412083402276, 0.0001950853766174987, 0.000164438781212084, 0.00014093428035266697, 0.00018844140868168324, 0.00017189000209327787, 0.00016026418597903103, 0.00013655464863404632, 0.00018111034296453, 0.00014381477376446128, 0.0001345233031315729, 0.00013078533811494708, 0.00014274133718572557, 0.00014392216689884663, 0.00011242362961638719, 0.00010264343291055411, 0.0001324669283349067, 0.00011476465442683548, 0.00010308727360097691, 9.641810174798593e-05, 0.00010602219845168293, 0.00010317425767425448, 9.570685506332666e-05, 8.980527491075918e-05, 8.832637104205787e-05, 8.974187221610919e-05, 9.015315299620852e-05, 7.656779052922502e-05, 8.714834257261828e-05, 7.817049481673166e-05, 7.527579873567447e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.14759047608822584, "first_to_last_ratio": 1.9719337947467823, "imbalance_abs_log_ratio": 0.6790146829732503, "last_quartile_mean": 0.07484555337578058, "mean": 0.1104847895912826, "normalized": [ 0.8984334974418968, 1.5927952138130326, 1.565859646322909, 1.3198735575362357, 1.1312139236939245, 1.5125315007911495, 1.3796810264894532, 1.286366010451999, 1.0960606168617568, 1.4536884922319049, 1.1543342013148132, 1.0797558953237962, 1.0497530983161607, 1.1457182562234747, 1.1551962948455465, 0.9023721940039697, 0.8238711061937815, 1.0632504861804335, 0.9211625151546727, 0.8274335809310764, 0.7739032642161865, 0.8509909070635955, 0.8281317391228538, 0.7681943896830757, 0.7208251689669645, 0.7089546589522648, 0.7203162343708721, 0.7236173971219663, 0.6145740294939214, 0.6994991437375198, 0.6274381319743458, 0.6042038211744466 ], "peak_layer": 2, "peak_normalized": 1.5927952138130326, "population_cv": 0.28896658018674404, "rest_mean": 0.11481397047086998, "spike_contrast": 0.7586813040631772, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.08710721284151077, "top_five_layers": [ 2, 3, 6, 10, 7 ], "top_five_spike_overlap": 0 }, "values": [ 0.09926323592662811, 0.1759796440601349, 0.1730036735534668, 0.14582595229148865, 0.12498193234205246, 0.16711172461509705, 0.15243376791477203, 0.14212387800216675, 0.12109802663326263, 0.16061046719551086, 0.1275363713502884, 0.1192966029047966, 0.11598175019025803, 0.12658444046974182, 0.12763161957263947, 0.09969840198755264, 0.0910252258181572, 0.11747300624847412, 0.10177444666624069, 0.09141882508993149, 0.08550453931093216, 0.09402155131101608, 0.09149596095085144, 0.08487379550933838, 0.0796402171254158, 0.0783287063241005, 0.07958398759365082, 0.07994871586561203, 0.06790108233690262, 0.07728401571512222, 0.069322369992733, 0.06675533205270767 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 5.093040158499207e-05, "first_to_last_ratio": 0.8860093039525244, "imbalance_abs_log_ratio": 0.12102782735710801, "last_quartile_mean": 5.748291960117058e-05, "mean": 5.7093861016710434e-05, "normalized": [ 0.4337136137178416, 0.9185569667519601, 0.9444095926066538, 0.8728787927927439, 0.7849140301431445, 1.1296292591692207, 1.0726593944235638, 0.9796135302182263, 0.9386642177229508, 1.3046353648934683, 1.0461832118506482, 1.0032917361864164, 0.9662905330123907, 1.0760080966276753, 1.1828739682986207, 0.9087955578115051, 0.9194941496879511, 1.198292373503509, 1.0946132296674007, 1.0673695485587213, 0.9244260207272295, 1.0525679458300632, 1.125653431968099, 0.9999504891362015, 0.9629027708994021, 1.0063234987710272, 0.9975933861195996, 1.0791525148711678, 0.9412468232151083, 1.085173475571988, 1.0031556955576197, 0.9789667796878816 ], "peak_layer": 10, "peak_normalized": 1.3046353648934683, "population_cv": 0.1441687540611157, "rest_mean": 5.695535414647397e-05, "spike_contrast": 1.0155638391297448, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.784179811598733e-05, "top_five_layers": [ 10, 18, 15, 6, 23 ], "top_five_spike_overlap": 1 }, "values": [ 2.4762384782661684e-05, 5.2443963795667514e-05, 5.3919990023132414e-05, 4.9836020480142906e-05, 4.481377254705876e-05, 6.449489592341706e-05, 6.124226638348773e-05, 5.592991874436848e-05, 5.3591964388033375e-05, 7.448667020071298e-05, 5.973063889541663e-05, 5.728179894504137e-05, 5.516925739357248e-05, 6.143345672171563e-05, 6.75348419463262e-05, 5.1886647270293906e-05, 5.249747118796222e-05, 6.84151382301934e-05, 6.249569560168311e-05, 6.094024865888059e-05, 5.277905074763112e-05, 6.009516800986603e-05, 6.426790059776977e-05, 5.7091034250333905e-05, 5.4975836974335834e-05, 5.74548939766828e-05, 5.695645813830197e-05, 6.161298369988799e-05, 5.373941530706361e-05, 6.19567435933277e-05, 5.727403186028823e-05, 5.58929932594765e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 5.801741053801379e-05, "first_to_last_ratio": 0.954195857595535, "imbalance_abs_log_ratio": 0.046886327142486896, "last_quartile_mean": 6.08024129178375e-05, "mean": 6.296396202287724e-05, "normalized": [ 0.4796364558374551, 0.9568353419130547, 0.9804638447804884, 0.8949545354119591, 0.8237377234766136, 1.1597846916175047, 1.085649359633547, 0.9904450076993521, 0.9595078978373608, 1.2941748206155441, 1.0680753760655664, 1.0231559828609242, 0.9904019047543832, 1.096490963646882, 1.1850351594596897, 0.9422215946487361, 0.9418631353047864, 1.1705024179981143, 1.0926353878857977, 1.0382275689133567, 0.9320833313178482, 1.0784583323883348, 1.0985080196909318, 0.9917906747078287, 0.9476388143731139, 0.9855066582268529, 0.978396290111904, 1.028407839812513, 0.8982055143685478, 1.0282804954549911, 0.9451124964564701, 0.9138123627295472 ], "peak_layer": 10, "peak_normalized": 1.2941748206155441, "population_cv": 0.13431233496292436, "rest_mean": 6.285090866084729e-05, "spike_contrast": 1.0115120295379672, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.357445017783903e-05, "top_five_layers": [ 10, 15, 18, 6, 23 ], "top_five_spike_overlap": 1 }, "values": [ 3.019981159013696e-05, 6.0246144130360335e-05, 6.173388828756288e-05, 5.634988337988034e-05, 5.186579073779285e-05, 7.302463927771896e-05, 6.835678505012766e-05, 6.236234185053036e-05, 6.041441884008236e-05, 8.148637425620109e-05, 6.725025741616264e-05, 6.442195444833487e-05, 6.235962791834027e-05, 6.903941539349034e-05, 7.461450877599418e-05, 5.9326004702597857e-05, 5.930343468207866e-05, 7.369946979451925e-05, 6.879665306769311e-05, 6.537092122016475e-05, 5.8687659475253895e-05, 6.790400948375463e-05, 6.916641723364592e-05, 6.244707037694752e-05, 5.966709431959316e-05, 6.205140380188823e-05, 6.160370685392991e-05, 6.475263216998428e-05, 5.655457789544016e-05, 6.474461406469345e-05, 5.950802733423188e-05, 5.753724690293893e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 1.7888660295284353e-05, "first_to_last_ratio": 0.41855462748958633, "imbalance_abs_log_ratio": 0.8709478659353102, "last_quartile_mean": 4.2739129185065394e-05, "mean": 3.1986895351110434e-05, "normalized": [ 0.2593955501951102, 0.4997829486650695, 0.48480994473028793, 0.4968760363401999, 0.5474125689219601, 0.7788406177150305, 0.7554473604850604, 0.6514322393717875, 0.7578135841604356, 0.9674810188795085, 0.9032772556875033, 0.9019220651946442, 0.8311356963419073, 0.9587970177975769, 1.1401394869948454, 0.8953886492354074, 1.0262272842074007, 1.1413916918814533, 1.174037958462901, 1.194927715786966, 1.0267140632187253, 1.3434898267078448, 1.3729067467361273, 1.2011929483563188, 1.1735959904166378, 1.3266745689353452, 1.2739952988364465, 1.4049535244931144, 1.2752889026856997, 1.4680979746684197, 1.4182292846928506, 1.3483241791974143 ], "peak_layer": 30, "peak_normalized": 1.4680979746684197, "population_cv": 0.3184955307507873, "rest_mean": 3.066251991657299e-05, "spike_contrast": 1.276428773763581, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.913852269761264e-05, "top_five_layers": [ 30, 31, 28, 23, 32 ], "top_five_spike_overlap": 1 }, "values": [ 8.297258318634704e-06, 1.5986504877218977e-05, 1.5507564967265353e-05, 1.589352177688852e-05, 1.7510028555989265e-05, 2.491269333404489e-05, 2.416441566310823e-05, 2.083729486912489e-05, 2.4240103812189773e-05, 3.094671410508454e-05, 2.889303505071439e-05, 2.8849686714238487e-05, 2.6585450541460887e-05, 3.066893987124786e-05, 3.6469522456172854e-05, 2.8640703021665104e-05, 3.282582474639639e-05, 3.6509576602838933e-05, 3.755382931558415e-05, 3.822202779701911e-05, 3.2841395295690745e-05, 4.2974068492185324e-05, 4.391502443468198e-05, 3.842243313556537e-05, 3.7539692129939795e-05, 4.243620060151443e-05, 4.0751154301688075e-05, 4.494010136113502e-05, 4.0792532672639936e-05, 4.695989628089592e-05, 4.5364751713350415e-05, 4.3128704419359565e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00022091567916504573, "first_to_last_ratio": 1.232986382535498, "imbalance_abs_log_ratio": 0.20943917994887448, "last_quartile_mean": 0.00017917122386279516, "mean": 0.0002198539991695725, "normalized": [ 0.4672441326905094, 1.0299890566632133, 1.08986441571188, 1.0084939820045402, 0.8814237539069011, 1.2693826047962031, 1.1886514905905403, 1.1035827467935222, 1.0473307057033197, 1.4274044616309502, 1.1442998336482333, 1.0911283607583675, 1.1094038041384326, 1.1946873974128753, 1.2511466100570947, 0.9923165311956169, 0.9239703707694655, 1.2473578196113966, 1.1181336030529359, 1.0160745414007064, 0.9397394354798955, 0.9905446518023159, 1.0121913654482901, 0.9359943959598221, 0.8839446941554127, 0.8510229499852975, 0.8648513533654478, 0.8786898836618169, 0.7455559493639876, 0.8399702482950041, 0.7392957942269532, 0.7163130557190521 ], "peak_layer": 10, "peak_normalized": 1.4274044616309502, "population_cv": 0.18952709251819305, "rest_mean": 0.0002217885959436427, "spike_contrast": 0.9441746799407325, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00020940717658959329, "top_five_layers": [ 10, 6, 15, 18, 14 ], "top_five_spike_overlap": 0 }, "values": [ 0.00010272549116052687, 0.00022644721320830286, 0.00023961105034686625, 0.00022172143508214504, 0.0001937845372594893, 0.0002790788421407342, 0.0002613297838252038, 0.00024262708029709756, 0.00023025984410196543, 0.000313820579322055, 0.00025157889467664063, 0.00023988893372006714, 0.00024390686303377151, 0.000262656802078709, 0.00027506958576850593, 0.0002181647578254342, 0.00020313858112785965, 0.00027423660503700376, 0.0002458261442370713, 0.00022338805138133466, 0.00020660547306761146, 0.00021777520305477083, 0.00022253431961871684, 0.00020578211115207523, 0.0001943387760547921, 0.00018710079893935472, 0.0001901410287246108, 0.00019318348495289683, 0.00016391345707233995, 0.00018467081827111542, 0.00016253713693004102, 0.00015748428995721042 ] } } }, "pre_mlp_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "2999a257265024350fd2665e2a8d452a03cd662193b4245d15290408ad29922d", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.014061377383768559, 0.014165290631353855, 0.014281988143920898, 0.014350161887705326, 0.00966146495193243, 0.009741888381540775, 0.009820535778999329, 0.00991273857653141, 0.0074927168898284435, 0.007546746172010899, 0.00766373984515667, 0.007762390188872814, 0.006325984839349985, 0.006375366821885109, 0.006445344537496567, 0.006555322092026472, 0.005531130358576775, 0.005611324217170477, 0.0056777093559503555, 0.00574630219489336, 0.004946626722812653, 0.0050392793491482735, 0.005084683652967215, 0.005203457549214363, 0.004667261149734259, 0.004717131145298481, 0.004768412560224533, 0.004826710093766451, 0.0044052014127373695, 0.004504534415900707, 0.004574235528707504, 0.004644096828997135 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 3.729895206561196e-05, "first_to_last_ratio": 1.2469721805352876, "imbalance_abs_log_ratio": 0.22071835733540474, "last_quartile_mean": 2.9911615229138988e-05, "mean": 3.340667609563752e-05, "normalized": [ 1.308752778077389, 1.2456464596610868, 1.0236956931411572, 0.9922285647386262, 1.26903181062375, 1.1170846373362797, 1.0287316533628064, 0.9469237624012169, 1.1300261751959062, 1.1132698789554223, 1.0126240729566007, 0.9379202574439188, 1.1130503370514597, 1.070713488608195, 0.9897642722064988, 0.922085688720865, 1.036831181462568, 0.9898105546019274, 0.9395130252451869, 0.9001967292772246, 0.9975952535130206, 0.9404117749145336, 0.9224862220158919, 0.8885787590271026, 0.9461670180110204, 0.8974372095121397, 0.8904691498559152, 0.8684437934704528, 0.9244147277585869, 0.9015689206268677, 0.8740901912628669, 0.8604359589635168 ], "peak_layer": 1, "peak_normalized": 1.308752778077389, "population_cv": 0.1164905978879517, "rest_mean": 3.378375206213807e-05, "spike_contrast": 0.9285666618330386, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.137046587653458e-05, "top_five_layers": [ 1, 5, 2, 9, 6 ], "top_five_spike_overlap": 0 }, "values": [ 4.3721080146497115e-05, 4.161290780757554e-05, 3.419827044126578e-05, 3.314705827506259e-05, 4.239413465256803e-05, 3.73180846509058e-05, 3.4366505133220926e-05, 3.1633575417799875e-05, 3.7750418414361775e-05, 3.719064625329338e-05, 3.3828404411906376e-05, 3.1332798243965954e-05, 3.718331208801828e-05, 3.5768978705164045e-05, 3.306473445263691e-05, 3.080381793552078e-05, 3.463708344497718e-05, 3.3066280593629926e-05, 3.138600732199848e-05, 3.007258055731654e-05, 3.332634150865488e-05, 3.1416031561093405e-05, 3.0817198421573266e-05, 2.9684462788281962e-05, 3.1608295103069395e-05, 2.9980394174344838e-05, 2.9747614462394267e-05, 2.9011820515734144e-05, 3.0881623388268054e-05, 3.01184209092753e-05, 2.9200447897892445e-05, 2.874430538213346e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 6.705871965095866e-05, "first_to_last_ratio": 0.7180297289018083, "imbalance_abs_log_ratio": 0.33124430563536084, "last_quartile_mean": 9.33926785364747e-05, "mean": 8.228966771639534e-05, "normalized": [ 0.8685826408392191, 0.8249006353131848, 0.7154905339803773, 0.6884730939760333, 0.9470487095509537, 0.8638069620260335, 0.8256155016985178, 0.7853664463613442, 0.9851686390305121, 0.9692511243547932, 0.9083804947955202, 0.8729855457896263, 1.0422548534804916, 1.0255492616133721, 0.9871528463964067, 0.9398223255303129, 1.0958375595009802, 1.0726325633807163, 1.0495259774381736, 1.0230838788771643, 1.1576441025653552, 1.1142532603054807, 1.1008567438213033, 1.056908774400763, 1.159063578749664, 1.1192434432436902, 1.1032241584740294, 1.1060435702772593, 1.1786235078649367, 1.1628365878275773, 1.141840825041133, 1.1085318534950752 ], "peak_layer": 29, "peak_normalized": 1.1786235078649367, "population_cv": 0.1341844256762087, "rest_mean": 8.049536754023629e-05, "spike_contrast": 1.1426606459269575, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.197888866765424e-05, "top_five_layers": [ 29, 30, 25, 21, 31 ], "top_five_spike_overlap": 2 }, "values": [ 7.14753768988885e-05, 6.788079917896539e-05, 5.887747829547152e-05, 5.66542221349664e-05, 7.793232362018898e-05, 7.108238787623122e-05, 6.793962529627606e-05, 6.462754390668124e-05, 8.106919995043427e-05, 7.975935295689851e-05, 7.475032907677814e-05, 7.183769048424438e-05, 8.576680556870997e-05, 8.4392107964959e-05, 8.123247971525416e-05, 7.733766688033938e-05, 9.017610864248127e-05, 8.826657722238451e-05, 8.636514394311234e-05, 8.418923243880272e-05, 9.526214853394777e-05, 9.169153054244816e-05, 9.0589135652408e-05, 8.697267185198143e-05, 9.537895675748587e-05, 9.210217103827745e-05, 9.078394941752777e-05, 9.101595787797123e-05, 9.698853682493791e-05, 9.56894364207983e-05, 9.396170207764953e-05, 9.122071787714958e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.05946834245696664, "first_to_last_ratio": 0.7180297222365345, "imbalance_abs_log_ratio": 0.33124431491808765, "last_quartile_mean": 0.08282156102359295, "mean": 0.07297530013602227, "normalized": [ 0.8685826122359729, 0.8249005917343738, 0.71549052911611, 0.6884731350635278, 0.9470486699630107, 0.8638070113577097, 0.8256154769796886, 0.7853664269052657, 0.9851686341386658, 0.9692511556795396, 0.9083805454214465, 0.8729855579949275, 1.0422548040385637, 1.0255492365750694, 0.9871528950273681, 0.9398223256645052, 1.0958375053304956, 1.07263252629438, 1.0495259690486791, 1.0230837911446407, 1.1576441435704576, 1.1142533045483842, 1.1008568124470575, 1.0569088284972943, 1.1590636022344982, 1.119243411839159, 1.1032241423789504, 1.1060435591240758, 1.1786235037173105, 1.1628366076958818, 1.141840809884862, 1.1085318743481278 ], "peak_layer": 29, "peak_normalized": 1.1786235037173105, "population_cv": 0.13418442668798974, "rest_mean": 0.07138409617322462, "spike_contrast": 1.1426607032635474, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.08156780153512955, "top_five_layers": [ 29, 30, 25, 21, 31 ], "top_five_spike_overlap": 2 }, "values": [ 0.06338507682085037, 0.0601973682641983, 0.05221313610672951, 0.050241533666849136, 0.0691111609339714, 0.06303657591342926, 0.060249537229537964, 0.05731235072016716, 0.07189297676086426, 0.07073139399290085, 0.06628934293985367, 0.06370638310909271, 0.07605885714292526, 0.0748397633433342, 0.07203777879476547, 0.06858381628990173, 0.07996907085180283, 0.07827568054199219, 0.07658947259187698, 0.07465984672307968, 0.0844794288277626, 0.08131296932697296, 0.0803353562951088, 0.07712823897600174, 0.08458301424980164, 0.08167712390422821, 0.08050811290740967, 0.08071386069059372, 0.0860104039311409, 0.08485835045576096, 0.08332617580890656, 0.08089544624090195 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 7.408341275549901e-06, "first_to_last_ratio": 0.49124437919492936, "imbalance_abs_log_ratio": 0.7108135577065074, "last_quartile_mean": 1.5080765479069669e-05, "mean": 1.1844782648040564e-05, "normalized": [ 0.5657990507718705, 0.5382233871697658, 0.49942145823009426, 0.48906984415931193, 0.8062921756393574, 0.7122953827323945, 0.7013508394120195, 0.6911626993599347, 0.8940675881372303, 0.868722127857376, 0.9735911331661544, 0.854855860776803, 0.989483587379368, 1.0059088486455374, 1.0303729006301217, 0.9941025538400318, 1.1620912657632405, 1.127012997009919, 1.113401270263767, 1.066097770938846, 1.1826034542812638, 1.1643547935214302, 1.2049778184223279, 1.1791491538213539, 1.2365172470752577, 1.219921447624247, 1.2337761202582145, 1.2288278254433027, 1.3039217543779833, 1.3631114810840155, 1.2856635723507386, 1.3138525898567213 ], "peak_layer": 30, "peak_normalized": 1.3631114810840155, "population_cv": 0.2549101540724733, "rest_mean": 1.1420299651271543e-05, "spike_contrast": 1.237882653019464, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.4136990830593277e-05, "top_five_layers": [ 30, 32, 29, 31, 25 ], "top_five_spike_overlap": 1 }, "values": [ 6.701766778860474e-06, 6.3751390371180605e-06, 5.915538622502936e-06, 5.7929260037781205e-06, 9.550355571263935e-06, 8.436983989668079e-06, 8.307348252856173e-06, 8.186671948351432e-06, 1.0590036254143342e-05, 1.0289824786013924e-05, 1.1531975360412616e-05, 1.0125581866304856e-05, 1.1720218026312068e-05, 1.1914771675947122e-05, 1.220454305439489e-05, 1.1774928680097219e-05, 1.3764718460151926e-05, 1.334922399109928e-05, 1.3187996046326589e-05, 1.2627696378331166e-05, 1.4007680874783546e-05, 1.379152945446549e-05, 1.4272700354922563e-05, 1.3966765436634887e-05, 1.4646278032159898e-05, 1.4449704394792207e-05, 1.4613809980801307e-05, 1.4555198504240252e-05, 1.5444669770658948e-05, 1.614575921848882e-05, 1.5228405572997872e-05, 1.5562298358418047e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 5.6029601637419546e-05, "first_to_last_ratio": 0.6070213107493427, "imbalance_abs_log_ratio": 0.49919138022039955, "last_quartile_mean": 9.230252817360451e-05, "mean": 7.840498903988191e-05, "normalized": [ 0.6598133090619259, 0.6419677920829977, 0.6005256373772865, 0.5913463573822231, 0.8562376513989928, 0.8062968559250673, 0.7923156566726156, 0.768439227921366, 0.9790122729268586, 0.9665730330981341, 0.9163710065350301, 0.8881686280668529, 1.0631190324760253, 1.0511030519715439, 1.0188080221730096, 0.9712812246823179, 1.1361272291219966, 1.1138561419734825, 1.0918134348286874, 1.0639217496772315, 1.203414696914147, 1.1592426077267617, 1.1442664103435203, 1.097952968622282, 1.2042281788778202, 1.1625445130141736, 1.1452986211445626, 1.1473932954411665, 1.2233942816523278, 1.205318760675343, 1.1829469858778916, 1.146901364356358 ], "peak_layer": 29, "peak_normalized": 1.2233942816523278, "population_cv": 0.1898597249088972, "rest_mean": 7.605543874820728e-05, "spike_contrast": 1.1977126437531995, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.109256061492487e-05, "top_five_layers": [ 29, 30, 25, 21, 31 ], "top_five_spike_overlap": 2 }, "values": [ 5.173265526536852e-05, 5.033347770222463e-05, 4.708420601673424e-05, 4.636450466932729e-05, 6.713330367347226e-05, 6.321769615169615e-05, 6.212150037754327e-05, 6.024946924299002e-05, 7.675944652874023e-05, 7.578414806630462e-05, 7.184805872384459e-05, 6.963685154914856e-05, 8.335383608937263e-05, 8.241172326961532e-05, 7.987963181221858e-05, 7.615329377586022e-05, 8.907804294722155e-05, 8.733187860343605e-05, 8.560362039133906e-05, 8.341677312273532e-05, 9.435371612198651e-05, 9.089040395338088e-05, 8.971619536168873e-05, 8.608499047113582e-05, 9.441749716643244e-05, 9.114928980125114e-05, 8.97971258382313e-05, 8.996135875349864e-05, 9.592021524440497e-05, 9.450300422031432e-05, 9.274894546251744e-05, 8.992278890218586e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 4.8511469685763586e-05, "first_to_last_ratio": 0.5271352332510719, "imbalance_abs_log_ratio": 0.6402981537748061, "last_quartile_mean": 9.202850924339145e-05, "mean": 7.532146378252946e-05, "normalized": [ 0.548104082267993, 0.5416830199005969, 0.528884702689083, 0.5247448675895723, 0.7777813723908718, 0.7487613756360456, 0.7496059866168319, 0.7329069161583202, 0.9480676391902623, 0.9399123881532756, 0.9000936195631334, 0.883441930780847, 1.0560589316963704, 1.0534552095740957, 1.032095781768693, 0.9835620691098637, 1.1601754560133901, 1.1467788545782247, 1.1233340534265408, 1.0967375255670042, 1.2396083970871608, 1.1951467072205837, 1.1828066060767042, 1.1317734013234224, 1.244698956870482, 1.2025862582796845, 1.1881970083126314, 1.1882256015362047, 1.2689460104605979, 1.2589162611011824, 1.2307315535498626, 1.1921774555104672 ], "peak_layer": 29, "peak_normalized": 1.2689460104605979, "population_cv": 0.23486334953575583, "rest_mean": 7.254842300123225e-05, "spike_contrast": 1.244629176846487, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.029588400153444e-05, "top_five_layers": [ 29, 30, 25, 21, 31 ], "top_five_spike_overlap": 2 }, "values": [ 4.1284001781605184e-05, 4.080035796505399e-05, 3.9836369978729635e-05, 3.952455153921619e-05, 5.858363147126511e-05, 5.639780283672735e-05, 5.6461420172126964e-05, 5.520362174138427e-05, 7.140984234865755e-05, 7.079557690303773e-05, 6.77963689668104e-05, 6.654213939327747e-05, 7.954390457598493e-05, 7.934778841445222e-05, 7.773896504659206e-05, 7.408333476632833e-05, 8.738611359149218e-05, 8.637706196168438e-05, 8.461116522084922e-05, 8.26078758109361e-05, 9.336911898571998e-05, 9.002019942272455e-05, 8.909072494134307e-05, 8.524682925781235e-05, 9.375254740007222e-05, 9.058055729838088e-05, 8.949673792812973e-05, 8.949889161158353e-05, 9.557887096889317e-05, 9.482341556577012e-05, 9.270050213672221e-05, 8.979655103757977e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 9.61669284151867e-05, "first_to_last_ratio": 0.8297833624837898, "imbalance_abs_log_ratio": 0.18659062131596024, "last_quartile_mean": 0.00011589401856326731, "mean": 0.00010716335873439675, "normalized": [ 0.9380742209455786, 0.8898820858744699, 0.7837417114644826, 0.7583102664636657, 1.0822070619439073, 0.9520749081713696, 0.9107982467806989, 0.8640017807748708, 1.0669538303069324, 1.0266950649381863, 0.9594029855154438, 0.8901506143407661, 1.0795289063552718, 1.0379165674077444, 0.9706971366471856, 0.914979822300863, 1.052939971267624, 1.0225707935964177, 0.9851238030379684, 0.9636652893153618, 1.0990044510799908, 1.0523904893769196, 1.0422234788441622, 1.0049019599938585, 1.0962247910521359, 1.0599378706264646, 1.05039930496957, 1.0583562345651827, 1.1199788667078585, 1.1100529546792268, 1.0850973923636869, 1.0717171382921347 ], "peak_layer": 29, "peak_normalized": 1.1199788667078585, "population_cv": 0.09156717959150794, "rest_mean": 0.00010599351123716958, "spike_contrast": 1.0706366257223145, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00011348053521942347, "top_five_layers": [ 29, 30, 21, 25, 31 ], "top_five_spike_overlap": 2 }, "values": [ 0.00010052718425868079, 9.536275319987908e-05, 8.398839418077841e-05, 8.12630751170218e-05, 0.00011597294360399246, 0.00010202754492638633, 9.760419925441965e-05, 9.25893327803351e-05, 0.00011433835607022047, 0.00011002409155480564, 0.00010281284630764276, 9.539152961224318e-05, 0.000115685943455901, 0.0001112266254494898, 0.0001040231654769741, 9.805231093196198e-05, 0.00011283658386673778, 0.00010958212078548968, 0.00010556917550275102, 0.00010326960909878835, 0.00011777300824178383, 0.0001127776995417662, 0.00011168816854478791, 0.00010768866923172027, 0.00011747513053705916, 0.00011358650226611644, 0.00011256431753281504, 0.00011341700883349404, 0.00012002069706795737, 0.00011895700299646705, 0.00011628268111962825, 0.00011484880815260112 ] } } } }, "selector": { "expected_uniform_count": 49, "identities_unique": true, "identity_order_sha256": "97fd9721a263024f6a746638f32e0ad9a988cd4c111c00e9e178c1022e5c9d5e", "passed": true, "uniform_count": 49, "uniform_indices": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 56, 57, 58, 59, 60, 61, 62, 63, 64 ], "visit_count": 65, "visits": [ { "branch": "attention", "coefficient": "uniform", "group": 1, "index": 0, "kind": "depth", "layer": 1, "offset": 1, "sources": 1, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 1, "index": 1, "kind": "depth", "layer": 1, "offset": 1, "sources": 2, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 1, "index": 2, "kind": "depth", "layer": 2, "offset": 2, "sources": 2, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 1, "index": 3, "kind": "depth", "layer": 2, "offset": 2, "sources": 2, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 1, "index": 4, "kind": "depth", "layer": 3, "offset": 3, "sources": 2, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 1, "index": 5, "kind": "depth", "layer": 3, "offset": 3, "sources": 2, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 1, "index": 6, "kind": "depth", "layer": 4, "offset": 4, "sources": 2, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 1, "index": 7, "kind": "depth", "layer": 4, "offset": 4, "sources": 2, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 2, "index": 8, "kind": "depth", "layer": 5, "offset": 1, "sources": 2, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 2, "index": 9, "kind": "depth", "layer": 5, "offset": 1, "sources": 3, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 2, "index": 10, "kind": "depth", "layer": 6, "offset": 2, "sources": 3, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 2, "index": 11, "kind": "depth", "layer": 6, "offset": 2, "sources": 3, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 2, "index": 12, "kind": "depth", "layer": 7, "offset": 3, "sources": 3, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 2, "index": 13, "kind": "depth", "layer": 7, "offset": 3, "sources": 3, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 2, "index": 14, "kind": "depth", "layer": 8, "offset": 4, "sources": 3, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 2, "index": 15, "kind": "depth", "layer": 8, "offset": 4, "sources": 3, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 3, "index": 16, "kind": "depth", "layer": 9, "offset": 1, "sources": 3, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 3, "index": 17, "kind": "depth", "layer": 9, "offset": 1, "sources": 4, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 3, "index": 18, "kind": "depth", "layer": 10, "offset": 2, "sources": 4, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 3, "index": 19, "kind": "depth", "layer": 10, "offset": 2, "sources": 4, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 3, "index": 20, "kind": "depth", "layer": 11, "offset": 3, "sources": 4, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 3, "index": 21, "kind": "depth", "layer": 11, "offset": 3, "sources": 4, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 3, "index": 22, "kind": "depth", "layer": 12, "offset": 4, "sources": 4, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 3, "index": 23, "kind": "depth", "layer": 12, "offset": 4, "sources": 4, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 4, "index": 24, "kind": "depth", "layer": 13, "offset": 1, "sources": 4, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 4, "index": 25, "kind": "depth", "layer": 13, "offset": 1, "sources": 5, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 4, "index": 26, "kind": "depth", "layer": 14, "offset": 2, "sources": 5, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 4, "index": 27, "kind": "depth", "layer": 14, "offset": 2, "sources": 5, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 4, "index": 28, "kind": "depth", "layer": 15, "offset": 3, "sources": 5, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 4, "index": 29, "kind": "depth", "layer": 15, "offset": 3, "sources": 5, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 4, "index": 30, "kind": "depth", "layer": 16, "offset": 4, "sources": 5, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 4, "index": 31, "kind": "depth", "layer": 16, "offset": 4, "sources": 5, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 5, "index": 32, "kind": "depth", "layer": 17, "offset": 1, "sources": 5, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 5, "index": 33, "kind": "depth", "layer": 17, "offset": 1, "sources": 6, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 5, "index": 34, "kind": "depth", "layer": 18, "offset": 2, "sources": 6, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 5, "index": 35, "kind": "depth", "layer": 18, "offset": 2, "sources": 6, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 5, "index": 36, "kind": "depth", "layer": 19, "offset": 3, "sources": 6, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 5, "index": 37, "kind": "depth", "layer": 19, "offset": 3, "sources": 6, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 5, "index": 38, "kind": "depth", "layer": 20, "offset": 4, "sources": 6, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 5, "index": 39, "kind": "depth", "layer": 20, "offset": 4, "sources": 6, "use_uniform": true }, { "branch": "attention", "coefficient": "detached_learned", "group": 6, "index": 40, "kind": "depth", "layer": 21, "offset": 1, "sources": 6, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 6, "index": 41, "kind": "depth", "layer": 21, "offset": 1, "sources": 7, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 6, "index": 42, "kind": "depth", "layer": 22, "offset": 2, "sources": 7, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 6, "index": 43, "kind": "depth", "layer": 22, "offset": 2, "sources": 7, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 6, "index": 44, "kind": "depth", "layer": 23, "offset": 3, "sources": 7, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 6, "index": 45, "kind": "depth", "layer": 23, "offset": 3, "sources": 7, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 6, "index": 46, "kind": "depth", "layer": 24, "offset": 4, "sources": 7, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 6, "index": 47, "kind": "depth", "layer": 24, "offset": 4, "sources": 7, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 7, "index": 48, "kind": "depth", "layer": 25, "offset": 1, "sources": 7, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 7, "index": 49, "kind": "depth", "layer": 25, "offset": 1, "sources": 8, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 7, "index": 50, "kind": "depth", "layer": 26, "offset": 2, "sources": 8, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 7, "index": 51, "kind": "depth", "layer": 26, "offset": 2, "sources": 8, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 7, "index": 52, "kind": "depth", "layer": 27, "offset": 3, "sources": 8, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 7, "index": 53, "kind": "depth", "layer": 27, "offset": 3, "sources": 8, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 7, "index": 54, "kind": "depth", "layer": 28, "offset": 4, "sources": 8, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 7, "index": 55, "kind": "depth", "layer": 28, "offset": 4, "sources": 8, "use_uniform": false }, { "branch": "attention", "coefficient": "uniform", "group": 8, "index": 56, "kind": "depth", "layer": 29, "offset": 1, "sources": 8, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 8, "index": 57, "kind": "depth", "layer": 29, "offset": 1, "sources": 9, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 8, "index": 58, "kind": "depth", "layer": 30, "offset": 2, "sources": 9, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 8, "index": 59, "kind": "depth", "layer": 30, "offset": 2, "sources": 9, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 8, "index": 60, "kind": "depth", "layer": 31, "offset": 3, "sources": 9, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 8, "index": 61, "kind": "depth", "layer": 31, "offset": 3, "sources": 9, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 8, "index": 62, "kind": "depth", "layer": 32, "offset": 4, "sources": 9, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 8, "index": 63, "kind": "depth", "layer": 32, "offset": 4, "sources": 9, "use_uniform": true }, { "branch": "output", "coefficient": "uniform", "group": null, "index": 64, "kind": "output", "layer": null, "offset": null, "sources": 9, "use_uniform": true } ], "visits_sha256": "dd2c48ef3a1efa7d1500720ce6e1f326e1c0b8cb70134a5b5e65d9d7680b9574" } }, "uniform_group_6_attention_only": { "bits_per_byte": 7.959341947457508, "forward": { "activation_sha256": { "attention_branch_output": "676759a2d7c0615ad7e1f246e90927fb829d00c8b6c3745032ae6b27ab7d902a", "mlp_branch_output": "afe29f3ef4d2992353f0bb59fe7c5c2c5050fca671a53184825a8d63cd83a2bc", "post_attention_state": "31853ffb07b24cc59e774bb20656840b4358b3080b42516d380f2dc0f46404c8", "post_mlp_state": "7fd9c911e93175896ad618842c120e77b3f017c17b7eec9c2bfb01fc6dbc8ac2", "pre_attention_input": "de71d4cf2d509a1063f6eac7f50c4885e9a058dee7e4b993ad40f5527f528d00", "pre_mlp_input": "2999a257265024350fd2665e2a8d452a03cd662193b4245d15290408ad29922d" }, "logits_sha256": "873a309a197bfd08fa523edcb440d2697903a34abc02dad1c7f5991509c3c151", "loss_nats": 5.516995429992676, "mixer_summary_sha256": "8450b24bdc777885a3357dc7b8080e7410acbe96943686433da76f379295133a" }, "loss_nats": 5.516995429992676, "loss_scale": 1.0, "mixers": [ { "branch": "attention", "entropy_mean": -0.0, "group": 1, "latest_source_mass": 1.0, "layer": 1, "max_source_mass": 1.0, "mixer_index": 0, "normalized_entropy": 1.0, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 1.0, "median": 1.0, "p05": 1.0, "p95": 1.0 } ], "sources": 1 }, { "branch": "mlp", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 1, "max_source_mass": 0.5, "mixer_index": 1, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 2, "max_source_mass": 0.5, "mixer_index": 2, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 2, "max_source_mass": 0.5, "mixer_index": 3, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 3, "max_source_mass": 0.5, "mixer_index": 4, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 3, "max_source_mass": 0.5, "mixer_index": 5, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 4, "max_source_mass": 0.5, "mixer_index": 6, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 4, "max_source_mass": 0.5, "mixer_index": 7, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6931471824645996, "group": 2, "latest_source_mass": 0.5, "layer": 5, "max_source_mass": 0.5, "mixer_index": 8, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "completed_group_1", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 5, "max_source_mass": 0.3333333432674408, "mixer_index": 9, "normalized_entropy": 1.0000000180537645, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 6, "max_source_mass": 0.3333333432674408, "mixer_index": 10, "normalized_entropy": 1.0000000180537645, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 6, "max_source_mass": 0.3333333432674408, "mixer_index": 11, "normalized_entropy": 1.0000000180537645, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 7, "max_source_mass": 0.3333333432674408, "mixer_index": 12, "normalized_entropy": 1.0000000180537645, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 7, "max_source_mass": 0.3333333432674408, "mixer_index": 13, "normalized_entropy": 1.0000000180537645, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 8, "max_source_mass": 0.3333333432674408, "mixer_index": 14, "normalized_entropy": 1.0000000180537645, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 8, "max_source_mass": 0.3333333432674408, "mixer_index": 15, "normalized_entropy": 1.0000000180537645, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0986123085021973, "group": 3, "latest_source_mass": 0.3333333432674408, "layer": 9, "max_source_mass": 0.3333333432674408, "mixer_index": 16, "normalized_entropy": 1.0000000180537645, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_2", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 9, "max_source_mass": 0.25, "mixer_index": 17, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 10, "max_source_mass": 0.25, "mixer_index": 18, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 10, "max_source_mass": 0.25, "mixer_index": 19, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 11, "max_source_mass": 0.25, "mixer_index": 20, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 11, "max_source_mass": 0.25, "mixer_index": 21, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 12, "max_source_mass": 0.25, "mixer_index": 22, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 12, "max_source_mass": 0.25, "mixer_index": 23, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3862943649291992, "group": 4, "latest_source_mass": 0.25, "layer": 13, "max_source_mass": 0.25, "mixer_index": 24, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_3", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 13, "max_source_mass": 0.20000000298023224, "mixer_index": 25, "normalized_entropy": 1.0000000186840277, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 14, "max_source_mass": 0.20000000298023224, "mixer_index": 26, "normalized_entropy": 1.0000000186840277, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 14, "max_source_mass": 0.20000000298023224, "mixer_index": 27, "normalized_entropy": 1.0000000186840277, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 15, "max_source_mass": 0.20000000298023224, "mixer_index": 28, "normalized_entropy": 1.0000000186840277, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 15, "max_source_mass": 0.20000000298023224, "mixer_index": 29, "normalized_entropy": 1.0000000186840277, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 16, "max_source_mass": 0.20000000298023224, "mixer_index": 30, "normalized_entropy": 1.0000000186840277, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 16, "max_source_mass": 0.20000000298023224, "mixer_index": 31, "normalized_entropy": 1.0000000186840277, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.6094379425048828, "group": 5, "latest_source_mass": 0.20000000298023224, "layer": 17, "max_source_mass": 0.20000000298023224, "mixer_index": 32, "normalized_entropy": 1.0000000186840277, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_4", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 17, "max_source_mass": 0.1666666716337204, "mixer_index": 33, "normalized_entropy": 1.0000001451965654, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 18, "max_source_mass": 0.1666666716337204, "mixer_index": 34, "normalized_entropy": 1.0000001451965654, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 18, "max_source_mass": 0.1666666716337204, "mixer_index": 35, "normalized_entropy": 1.0000001451965654, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 19, "max_source_mass": 0.1666666716337204, "mixer_index": 36, "normalized_entropy": 1.0000001451965654, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 19, "max_source_mass": 0.1666666716337204, "mixer_index": 37, "normalized_entropy": 1.0000001451965654, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 20, "max_source_mass": 0.1666666716337204, "mixer_index": 38, "normalized_entropy": 1.0000001451965654, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 20, "max_source_mass": 0.1666666716337204, "mixer_index": 39, "normalized_entropy": 1.0000001451965654, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.791759729385376, "group": 6, "latest_source_mass": 0.1666666716337204, "layer": 21, "max_source_mass": 0.1666666716337204, "mixer_index": 40, "normalized_entropy": 1.0000001451965654, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_5", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 21, "max_source_mass": 0.1428571492433548, "mixer_index": 41, "normalized_entropy": 1.00000003408302, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 22, "max_source_mass": 0.1428571492433548, "mixer_index": 42, "normalized_entropy": 1.00000003408302, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 22, "max_source_mass": 0.1428571492433548, "mixer_index": 43, "normalized_entropy": 1.00000003408302, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 23, "max_source_mass": 0.1428571492433548, "mixer_index": 44, "normalized_entropy": 1.00000003408302, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 23, "max_source_mass": 0.1428571492433548, "mixer_index": 45, "normalized_entropy": 1.00000003408302, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 24, "max_source_mass": 0.1428571492433548, "mixer_index": 46, "normalized_entropy": 1.00000003408302, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 24, "max_source_mass": 0.1428571492433548, "mixer_index": 47, "normalized_entropy": 1.00000003408302, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9459102153778076, "group": 7, "latest_source_mass": 0.1428571492433548, "layer": 25, "max_source_mass": 0.1428571492433548, "mixer_index": 48, "normalized_entropy": 1.00000003408302, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_6", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 25, "max_source_mass": 0.125, "mixer_index": 49, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 26, "max_source_mass": 0.125, "mixer_index": 50, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 26, "max_source_mass": 0.125, "mixer_index": 51, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 27, "max_source_mass": 0.125, "mixer_index": 52, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 27, "max_source_mass": 0.125, "mixer_index": 53, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 28, "max_source_mass": 0.125, "mixer_index": 54, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 28, "max_source_mass": 0.125, "mixer_index": 55, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.079441547393799, "group": 8, "latest_source_mass": 0.125, "layer": 29, "max_source_mass": 0.125, "mixer_index": 56, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_7", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 29, "max_source_mass": 0.1111111119389534, "mixer_index": 57, "normalized_entropy": 1.0000000180537645, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 30, "max_source_mass": 0.1111111119389534, "mixer_index": 58, "normalized_entropy": 1.0000000180537645, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 30, "max_source_mass": 0.1111111119389534, "mixer_index": 59, "normalized_entropy": 1.0000000180537645, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 31, "max_source_mass": 0.1111111119389534, "mixer_index": 60, "normalized_entropy": 1.0000000180537645, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 31, "max_source_mass": 0.1111111119389534, "mixer_index": 61, "normalized_entropy": 1.0000000180537645, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 32, "max_source_mass": 0.1111111119389534, "mixer_index": 62, "normalized_entropy": 1.0000000180537645, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 32, "max_source_mass": 0.1111111119389534, "mixer_index": 63, "normalized_entropy": 1.0000000180537645, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "output", "entropy_mean": 2.1972246170043945, "group": null, "latest_source_mass": 0.1111111119389534, "layer": null, "max_source_mass": 0.1111111119389534, "mixer_index": null, "normalized_entropy": 1.0000000180537645, "offset": null, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_8", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 } ], "mode": "uniform_group_6_attention_only", "optimizer_state_after": "b82a349700eda58e2330a22ca2ee1f7fb1406efd47fd9aa306122b368a114192", "optimizer_state_before": "b82a349700eda58e2330a22ca2ee1f7fb1406efd47fd9aa306122b368a114192", "optimizer_state_unchanged": true, "positions": { "attention_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "676759a2d7c0615ad7e1f246e90927fb829d00c8b6c3745032ae6b27ab7d902a", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.0022265110164880753, 0.0021613112185150385, 0.0025601957459002733, 0.0025323182344436646, 0.0027253651060163975, 0.002904455177485943, 0.0029208383057266474, 0.00299624796025455, 0.0030937574338167906, 0.0033804215490818024, 0.0034370317589491606, 0.0038969130255281925, 0.004255065228790045, 0.004119740333408117, 0.003970506135374308, 0.004385598469525576, 0.0041999719105660915, 0.005232699681073427, 0.004664790350943804, 0.004898969549685717, 0.005131533369421959, 0.005069608334451914, 0.0051318020559847355, 0.0058372304774820805, 0.0059690922498703, 0.00554921617731452, 0.0058550527319312096, 0.006255472078919411, 0.006502739619463682, 0.006122888531535864, 0.007071305997669697, 0.006241494789719582 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.00015962477027642308, "first_to_last_ratio": 3.7699040072151497, "imbalance_abs_log_ratio": 1.3270495388586014, "last_quartile_mean": 4.234186599205714e-05, "mean": 8.617327489446325e-05, "normalized": [ 2.5428430303860874, 2.3222064178412967, 2.1011513502014654, 1.8911470095267005, 1.7004447966680725, 1.5353700031893112, 1.4229335846259188, 1.3028620492588128, 1.2049362956920768, 1.1705197150696323, 1.0773612761873523, 1.0133200843661747, 0.9906112963967714, 0.9272530070793641, 0.8550207003106389, 0.8185600560490035, 0.7703816559177782, 0.7173918665339755, 0.6760157258502337, 0.6514535263006792, 0.6184138115448249, 0.6053606047045522, 0.5838821473278404, 0.5697017460729616, 0.5457761369570343, 0.5276086337508958, 0.5104655265174319, 0.4951752820428687, 0.47527333481689216, 0.465789239220051, 0.45896414036514044, 0.45180594922816086 ], "peak_layer": 1, "peak_normalized": 2.5428430303860874, "population_cv": 0.5705341151299013, "rest_mean": 9.280180475562466e-05, "spike_contrast": 0.5428688997682254, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.0379213644191625e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00021912511147093028, 0.00020011213200632483, 0.00018106309289578348, 0.00016296633111778647, 0.00014653289690613747, 0.00013230786134954542, 0.0001226188469445333, 0.00011227188952034339, 0.0001038333066389896, 0.00010086751717608422, 9.283974941354245e-05, 8.732111018616706e-05, 8.536421955795959e-05, 7.990442827576771e-05, 7.367993384832516e-05, 7.053800072753802e-05, 6.63863102090545e-05, 6.182000652188435e-05, 5.825448897667229e-05, 5.6137883802875876e-05, 5.329074338078499e-05, 5.216590579948388e-05, 5.031503678765148e-05, 4.909306517220102e-05, 4.703131708083674e-05, 4.5465763832908124e-05, 4.3988486140733585e-05, 4.2670875700423494e-05, 4.095585973118432e-05, 4.013858415419236e-05, 3.955044303438626e-05, 3.893359826179221e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.00021442661636683624, "first_to_last_ratio": 1.843144421770586, "imbalance_abs_log_ratio": 0.6114730379455638, "last_quartile_mean": 0.0001163373926829081, "mean": 0.00015221274247778638, "normalized": [ 1.7741446897870732, 1.648461275666186, 1.5301464224684427, 1.4217121808823485, 1.3265065496562456, 1.2454420544341354, 1.1897998831173078, 1.1336248235930768, 1.0869089599614004, 1.0643003125599004, 1.0227030055526172, 0.9941765648180964, 0.9788958472869692, 0.9511102301475627, 0.9228060651723192, 0.9052418824754537, 0.8846484396889256, 0.865292475366091, 0.8497311645646503, 0.8388691935254817, 0.8256303537004397, 0.8176325372729673, 0.8076852907347736, 0.8000670639426607, 0.7892588692968658, 0.7814415503461742, 0.7731202151361662, 0.7664994567889658, 0.758490072461763, 0.7529625284402992, 0.7485174912671357, 0.7441725498875049 ], "peak_layer": 1, "peak_normalized": 1.7741446897870732, "population_cv": 0.27364872432853743, "rest_mean": 0.00015762320577373935, "spike_contrast": 0.7803181014868814, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012299624067964033, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00027004742878489196, 0.0002509168116375804, 0.0002329077833564952, 0.00021640271006617695, 0.00020191119983792305, 0.00018957215070258826, 0.0001811027032090351, 0.00017255214333999902, 0.00016544139361940324, 0.00016200006939470768, 0.00015566842921543866, 0.0001513263414381072, 0.00014900042151566595, 0.0001447710965294391, 0.00014046284195501357, 0.0001377893495373428, 0.00013465476513374597, 0.00013170854072086513, 0.00012933991092722863, 0.0001276865805266425, 0.00012567146040964872, 0.00012445409083738923, 0.00012293999316170812, 0.00012178040196886286, 0.00012013525702059269, 0.00011894536146428436, 0.00011767874821089208, 0.00011667098442558199, 0.00011545185407157987, 0.00011461049143690616, 0.00011393390013836324, 0.00011327254469506443 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.19015566632151604, "first_to_last_ratio": 1.8431444167877922, "imbalance_abs_log_ratio": 0.6114730352421437, "last_quartile_mean": 0.10316916275769472, "mean": 0.13498378219082952, "normalized": [ 1.7741447141792424, 1.6484611693507762, 1.5301464164964773, 1.421712099711509, 1.3265065258282553, 1.245442086656075, 1.1897999853143832, 1.1336247998774232, 1.0869090145314644, 1.0643003539468234, 1.0227030148964142, 0.9941765578539046, 0.978895844117059, 0.9511102293531465, 0.9228060470726628, 0.9052419251832852, 0.8846484748200404, 0.8652925213936268, 0.8497311905993837, 0.8388692019864461, 0.8256303029992552, 0.8176324960789627, 0.8076852727084025, 0.800067049482784, 0.789258877083647, 0.7814415614650243, 0.7731201949085273, 0.7664994759042474, 0.758490077799444, 0.7529625178867759, 0.7485174642151933, 0.7441725362993381 ], "peak_layer": 1, "peak_normalized": 1.7741447141792424, "population_cv": 0.2736487184871982, "rest_mean": 0.13978183573042904, "spike_contrast": 0.7803180757143806, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.10907429307699204, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.23948076367378235, 0.2225155234336853, 0.20654495060443878, 0.1919080764055252, 0.17905686795711517, 0.16811448335647583, 0.16060370206832886, 0.1530209630727768, 0.14671508967876434, 0.14366328716278076, 0.13804832100868225, 0.13419771194458008, 0.1321350634098053, 0.12838445603847504, 0.12456385046243668, 0.12219297885894775, 0.11941319704055786, 0.116800457239151, 0.11469992995262146, 0.11323373764753342, 0.11144670099020004, 0.11036712676286697, 0.10902441293001175, 0.10799607634544373, 0.10653714835643768, 0.10548193752765656, 0.10435868799686432, 0.1034649983048439, 0.10238385945558548, 0.10163772851228714, 0.10103771835565567, 0.10045122355222702 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 6.883365585963475e-05, "first_to_last_ratio": 2.2497546817252747, "imbalance_abs_log_ratio": 0.8108211799277664, "last_quartile_mean": 3.059607183786284e-05, "mean": 4.691841348858361e-05, "normalized": [ 1.7359303646705315, 1.6475859007944922, 1.5536416135247637, 1.4921074478374423, 1.436565604934822, 1.343834173349448, 1.2787439414688093, 1.2483315192759759, 1.191227587726315, 1.1603086071531024, 1.0995306735006558, 1.059977946939621, 1.0583984121521266, 1.0176934649371145, 0.9754909410255406, 0.9631621800279858, 0.9242301483750994, 0.8752327792179452, 0.8454160845678106, 0.8232909667825862, 0.7863453139125789, 0.77746685672799, 0.7497386589361914, 0.7388508716017261, 0.7093180450413769, 0.6908354487677191, 0.6685471901846501, 0.6592354484095886, 0.631440334976849, 0.6235901145414114, 0.6199506557967245, 0.6139807028410055 ], "peak_layer": 1, "peak_normalized": 1.7359303646705315, "population_cv": 0.3220412433027962, "rest_mean": 4.907019663530971e-05, "spike_contrast": 0.7193528234378936, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.529878449626267e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 8.144709863699973e-05, 7.730211655143648e-05, 7.289439963642508e-05, 7.000731420703232e-05, 6.740137905580923e-05, 6.305056740529835e-05, 5.999663699185476e-05, 5.856973439222202e-05, 5.5890508519951254e-05, 5.4439839004771784e-05, 5.1588234782684594e-05, 4.9732483603293076e-05, 4.965837433701381e-05, 4.774856279254891e-05, 4.576848732540384e-05, 4.519004141911864e-05, 4.3363412260077894e-05, 4.106453343410976e-05, 3.966558142565191e-05, 3.862750600092113e-05, 3.6894074582960457e-05, 3.647751145763323e-05, 3.517654840834439e-05, 3.4665710700210184e-05, 3.328007733216509e-05, 3.241290323785506e-05, 3.136717350571416e-05, 3.0930281354812905e-05, 2.9626178729813546e-05, 2.925785884144716e-05, 2.9087101211189292e-05, 2.8807000489905477e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 0.0001734125216898974, "first_to_last_ratio": 1.495517223041696, "imbalance_abs_log_ratio": 0.4024721155990803, "last_quartile_mean": 0.00011595488103921525, "mean": 0.00013990532806928968, "normalized": [ 1.3768889945483427, 1.3340940553978193, 1.2910049686488183, 1.2527203273458163, 1.2147536525239127, 1.1769854337196524, 1.14744939519443, 1.1220955988146075, 1.0944022472959734, 1.0773022658134643, 1.0508559212107247, 1.03206209473486, 1.0208171898483585, 1.000561882500425, 0.9805161605170362, 0.9663129296065233, 0.9487424991891081, 0.931931984738623, 0.9181974345490227, 0.9075104531838204, 0.8943640941785878, 0.8862664021756573, 0.8757830774574916, 0.86790401991425, 0.8562894022841879, 0.8478155487572495, 0.8387227683262297, 0.8314011142413483, 0.8224071457687988, 0.8161875050523302, 0.8112199678685613, 0.8064334645939688 ], "peak_layer": 1, "peak_normalized": 1.3768889945483427, "population_cv": 0.16325957486062354, "rest_mean": 0.0001431148236776978, "spike_contrast": 0.8564734849544942, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001225740517838858, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0001926341064972803, 0.00018664686649572104, 0.00018061847367789596, 0.00017526224837638438, 0.0001699505082797259, 0.00016466653323732316, 0.00016053428407758474, 0.00015698715287726372, 0.00015311270544771105, 0.00015072032692842185, 0.00014702034241054207, 0.00014439098595175892, 0.00014281776384450495, 0.00013998393842484802, 0.00013717943511437625, 0.00013519232743419707, 0.00013273413060232997, 0.00013038225006312132, 0.00012846071331296116, 0.00012696554767899215, 0.0001251263020094484, 0.00012399339175317436, 0.0001225267187692225, 0.00012142439663875848, 0.00011979944974882528, 0.00011861391249112785, 0.00011734178406186402, 0.00011631744564510882, 0.00011505914153531194, 0.00011418898066040128, 0.00011349399574100971, 0.00011282433843007311 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 0.00014675117381557357, "first_to_last_ratio": 1.2676837465168667, "imbalance_abs_log_ratio": 0.23719141364257146, "last_quartile_mean": 0.00011576323686313117, "mean": 0.00013042020850662084, "normalized": [ 1.169988069421413, 1.156175152925158, 1.1417665260225645, 1.1306726331060184, 1.1183621029407895, 1.1062694829504214, 1.0940115056230622, 1.0844989956600726, 1.0711591662817939, 1.0579765491085218, 1.0443900226505625, 1.035471661013805, 1.0260498632741444, 1.013992390086173, 1.0062479316494695, 0.9977001177294099, 0.9863163473759529, 0.9769959732660659, 0.9687585669786032, 0.9602628595878113, 0.9503845997255937, 0.9426104617668664, 0.9330243108845692, 0.9259760938667744, 0.914797625470175, 0.906289421438531, 0.8956357003805738, 0.888470885240542, 0.8806216558595509, 0.8756991509238615, 0.8712858841028995, 0.8681382926882502 ], "peak_layer": 1, "peak_normalized": 1.169988069421413, "population_cv": 0.09123006364857297, "rest_mean": 0.00013202972385678786, "spike_contrast": 0.9219804606102014, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012172882561571896, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00015259008796419948, 0.00015078860451467335, 0.00014890942838974297, 0.0001474625605624169, 0.00014585701865144074, 0.00014427989663090557, 0.00014268120867200196, 0.00014144058513920754, 0.00013970080181024969, 0.0001379815221298486, 0.00013620956451632082, 0.00013504642993211746, 0.0001338176371064037, 0.00013224509893916547, 0.00013123506505507976, 0.0001301202573813498, 0.00012863558367826045, 0.00012742001854348928, 0.00012634569429792464, 0.00012523768236860633, 0.00012394935765769333, 0.00012293545296415687, 0.00012168522516731173, 0.00012076599523425102, 0.00011930809705518186, 0.00011819845531135798, 0.00011680899478960782, 0.00011587455810513347, 0.00011485085997264832, 0.00011420886585256085, 0.00011363328667357564, 0.00011322277714498341 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00030002388848515693, "first_to_last_ratio": 2.2836575149168885, "imbalance_abs_log_ratio": 0.8257783308640597, "last_quartile_mean": 0.0001313786706305109, "mean": 0.0001947244400071213, "normalized": [ 1.8998833386658438, 1.7835645331795598, 1.6713073472446418, 1.5703972875779748, 1.4780250735507643, 1.3761669788219328, 1.3066447563403325, 1.2401013786965793, 1.1762106459027857, 1.150547637187856, 1.0768735950666006, 1.0264900078881174, 1.0140784159549234, 0.9666871251601796, 0.9096841069789114, 0.8754807117769676, 0.8421876125778792, 0.8073809902734768, 0.77911667677541, 0.7607651063686129, 0.7403822030704592, 0.7308855611321243, 0.7152791523187013, 0.7043381880055872, 0.6944157299089508, 0.686864629891504, 0.680599049276497, 0.6764772714226681, 0.6702412094778135, 0.6672258960044359, 0.6635863560708457, 0.6581114274310631 ], "peak_layer": 1, "peak_normalized": 1.8998833386658438, "population_cv": 0.3592853832280775, "rest_mean": 0.00020492727344390005, "spike_contrast": 0.6813594750078023, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013962913944851608, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0003699537192005664, 0.0003473036049399525, 0.0003254443872720003, 0.00030579473241232336, 0.00028780760476365685, 0.00026797334430739284, 0.0002544356684666127, 0.0002414780465187505, 0.0002290369593538344, 0.00022403974435292184, 0.00020969360775779933, 0.00019988269195891917, 0.0001974658516701311, 0.00018823760910890996, 0.00017713772831484675, 0.00017047749133780599, 0.00016399451124016196, 0.0001572168112033978, 0.00015171305858530104, 0.00014813955931458622, 0.00014417050988413393, 0.00014232128160074353, 0.00013928233238402754, 0.0001371518592350185, 0.00013521971413865685, 0.00013374933041632175, 0.00013252926873974502, 0.00013172665785532445, 0.00013051234418526292, 0.00012992518895771354, 0.00012921648158226162, 0.000128150379168801 ] } } }, "mlp_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "afe29f3ef4d2992353f0bb59fe7c5c2c5050fca671a53184825a8d63cd83a2bc", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.0027073738165199757, 0.002722575329244137, 0.0026850115973502398, 0.0026861224323511124, 0.00271838391199708, 0.0026645169127732515, 0.0026859452482312918, 0.0027359756641089916, 0.0026767742820084095, 0.0027451682835817337, 0.0026999011170119047, 0.002663193503394723, 0.002626648638397455, 0.0026531771291047335, 0.0026517712976783514, 0.002676903735846281, 0.002670086920261383, 0.0026355870068073273, 0.0026610493659973145, 0.0026388936676084995, 0.0026469926815479994, 0.00264862016774714, 0.002641494618728757, 0.002674198243767023, 0.002557571977376938, 0.00252733426168561, 0.0025468147359788418, 0.0026856842450797558, 0.002551089273765683, 0.0025778315030038357, 0.0026001501828432083, 0.0025662274565547705 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.00015858857113926206, "first_to_last_ratio": 3.757867690304466, "imbalance_abs_log_ratio": 1.3238516928952826, "last_quartile_mean": 4.220174423608114e-05, "mean": 8.573221236929385e-05, "normalized": [ 2.5461641896458365, 2.3032502265274117, 2.0924715218207024, 1.8886722264368014, 1.7031663195050915, 1.537434754996766, 1.4223955235407997, 1.3049501874408618, 1.2055297161045848, 1.1692542365736316, 1.0769571962730058, 1.0144505847411098, 0.9913601784485884, 0.9317001386730132, 0.857757082878185, 0.8155634736467603, 0.7682612945963089, 0.7217806418748633, 0.678338533281248, 0.6511820384364709, 0.6192882306865346, 0.6053793146733001, 0.5848721334223365, 0.5718144052517429, 0.5462281883927541, 0.527515083157495, 0.5112993585545178, 0.496920907994256, 0.47722358366421574, 0.46552107689700334, 0.4601234027970099, 0.45317424906679377 ], "peak_layer": 1, "peak_normalized": 2.5461641896458365, "population_cv": 0.5686091737198813, "rest_mean": 9.231269224848668e-05, "spike_contrast": 0.5437781067692293, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.0197621021652594e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00021828828903380781, 0.00019746273756027222, 0.00017939221288543195, 0.0001619200484128669, 0.00014601621660403907, 0.000131807682919316, 0.00012194511509733275, 0.00011187626660102978, 0.00010335272963857278, 0.00010024275252362713, 9.232992306351662e-05, 8.697109296917915e-05, 8.499150135321543e-05, 7.987671415321529e-05, 7.353741239057854e-05, 6.992006092332304e-05, 6.586474046343938e-05, 6.187985127326101e-05, 5.8155463193543255e-05, 5.582727681030519e-05, 5.309295011102222e-05, 5.1900507969548926e-05, 5.0142381951445714e-05, 4.902291402686387e-05, 4.682935104938224e-05, 4.522503513726406e-05, 4.3834825191879645e-05, 4.260212881490588e-05, 4.0913433622336015e-05, 3.991015182691626e-05, 3.944739728467539e-05, 3.885163096128963e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.00021242323236947414, "first_to_last_ratio": 1.8345651540958352, "imbalance_abs_log_ratio": 0.6068074801791611, "last_quartile_mean": 0.0001157894184871111, "mean": 0.00015111673860701558, "normalized": [ 1.771707915810251, 1.6366668487510905, 1.5242909365839221, 1.4186705636795862, 1.3264449439106374, 1.2453857912414694, 1.1893415085703798, 1.1330085309323608, 1.0858463851390703, 1.062861239127963, 1.023237806768965, 0.9942126959777651, 0.978436257513136, 0.9531027445378829, 0.9232414968561398, 0.904013237249231, 0.8853704564278231, 0.8680164030054869, 0.8515402788738432, 0.8395817783841374, 0.8267435189825344, 0.8185143642831039, 0.809172703595369, 0.8007918347797284, 0.7909896391756025, 0.7821052393452608, 0.7753388667989797, 0.7682453290925442, 0.7604995795745633, 0.7549901567959042, 0.7508572350862155, 0.7467737131490535 ], "peak_layer": 1, "peak_normalized": 1.771707915810251, "population_cv": 0.2714170004285523, "rest_mean": 0.00015645500981783562, "spike_contrast": 0.7816309251520459, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012229007406858728, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0002677347220014781, 0.00024732775636948645, 0.00023034587502479553, 0.00021438486874103546, 0.00020044803386554122, 0.0001881986390799284, 0.00017972940986510366, 0.00017121655400842428, 0.00016408956435043365, 0.00016061612404882908, 0.0001546283601783216, 0.00015024218009784818, 0.00014785809617023915, 0.0001440297783119604, 0.00013951724395155907, 0.000136611532070674, 0.0001337942958343774, 0.00013117180787958205, 0.0001286819897359237, 0.00012687486014328897, 0.00012493478425312787, 0.00012369122123345733, 0.00012227953993715346, 0.00012101305037504062, 0.00011953177454415709, 0.00011818919301731512, 0.00011716668086592108, 0.00011609472858253866, 0.00011492421617731452, 0.00011409165017539635, 0.00011346709652571008, 0.00011285000800853595 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.18837904930114746, "first_to_last_ratio": 1.8345651581276443, "imbalance_abs_log_ratio": 0.6068074823768531, "last_quartile_mean": 0.10268321540206671, "mean": 0.13401183625683188, "normalized": [ 1.7717078885231814, 1.6366668119471057, 1.524290957183596, 1.418670514273678, 1.326444920903592, 1.2453858718827995, 1.1893415500975306, 1.1330085716119305, 1.0858463365841577, 1.062861212873729, 1.0232377434953237, 0.9942127340740732, 0.9784362443707608, 0.9531027271661117, 0.9232414917376033, 0.904013241653558, 0.885370477646398, 0.8680164390463408, 0.8515402681847718, 0.8395816969955481, 0.8267435345782925, 0.8185144279720658, 0.8091727254708025, 0.8007918405918669, 0.7909896328038464, 0.7821052665924377, 0.7753388466443417, 0.7682453528580236, 0.7604996016741158, 0.7549901615676585, 0.7508572336421532, 0.7467736753526057 ], "peak_layer": 1, "peak_normalized": 1.7717078885231814, "population_cv": 0.27141699472708714, "rest_mean": 0.1387458680956452, "spike_contrast": 0.7816309473985974, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.10844806432724, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.23742982745170593, 0.2193327248096466, 0.2042730301618576, 0.19011864066123962, 0.1777593195438385, 0.16689644753932953, 0.15938584506511688, 0.151836559176445, 0.1455162614583969, 0.1424359828233719, 0.13712596893310547, 0.13323627412319183, 0.13112203776836395, 0.12772704660892487, 0.1237252876162529, 0.1211484745144844, 0.11865012347698212, 0.11632447689771652, 0.11411647498607635, 0.11251388490200043, 0.11079341918230057, 0.10969062149524689, 0.10843872278928757, 0.10731558501720428, 0.10600197315216064, 0.10481136292219162, 0.10390458256006241, 0.1029539704322815, 0.10191594809293747, 0.10117761790752411, 0.10062375664710999, 0.10007651150226593 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 6.873187476230669e-05, "first_to_last_ratio": 2.248757500145603, "imbalance_abs_log_ratio": 0.8103778415280068, "last_quartile_mean": 3.0564378221242805e-05, "mean": 4.685761763312257e-05, "normalized": [ 1.7400006587426948, 1.6405194725375354, 1.5551306973978087, 1.4944078341288562, 1.4327724019957897, 1.3414187773842097, 1.2804088050749736, 1.2499327782691996, 1.1927955171655855, 1.1608931065609767, 1.0981756972550076, 1.0577423904261343, 1.0542786038595535, 1.0218640087393278, 0.9775807389332813, 0.9570972402588515, 0.9209982861589985, 0.8815092249082567, 0.8478232134430647, 0.8204952153707635, 0.7871780668180614, 0.7759097745574741, 0.7506963524238334, 0.7421155343193898, 0.7072927325438048, 0.6891814936654451, 0.6695399899996352, 0.6616343983211914, 0.6345111343049281, 0.6212687927046295, 0.6200231913288072, 0.6148038704019305 ], "peak_layer": 1, "peak_normalized": 1.7400006587426948, "population_cv": 0.3217536161653951, "rest_mean": 4.900405669845727e-05, "spike_contrast": 0.719671983430414, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.526684668031521e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 8.153228554874659e-05, 7.687083416385576e-05, 7.286971958819777e-05, 7.002439087955281e-05, 6.71363013680093e-05, 6.285568815656006e-05, 5.999690620228648e-05, 5.8568872191244736e-05, 5.5891556257847697e-05, 5.439668530016206e-05, 5.145789691596292e-05, 4.9563288484932855e-05, 4.9400983698433265e-05, 4.7882112994557247e-05, 4.580710447044112e-05, 4.484729652176611e-05, 4.3155785533599555e-05, 4.130542220082134e-05, 3.972697595600039e-05, 3.844645107164979e-05, 3.688528886414133e-05, 3.635728353401646e-05, 3.517584264045581e-05, 3.477376594673842e-05, 3.314205241622403e-05, 3.229340290999971e-05, 3.1373048841487616e-05, 3.10026116494555e-05, 2.9731680115219206e-05, 2.9111175535945222e-05, 2.9052809622953646e-05, 2.8808244678657502e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 0.0001719075116852764, "first_to_last_ratio": 1.4894635014227988, "imbalance_abs_log_ratio": 0.39841598895878805, "last_quartile_mean": 0.00011541572621354135, "mean": 0.00013894635094402474, "normalized": [ 1.3738985779819257, 1.3296082759737424, 1.2883384004275333, 1.2499940655466777, 1.2130730176151365, 1.1755503979222366, 1.147409009170091, 1.11990595257071, 1.0929037170303153, 1.075598370036652, 1.0512762916941902, 1.0312594730951, 1.0194418974669357, 1.0018996501730997, 0.9803460168062212, 0.9648004557293595, 0.9493010807852901, 0.9344788929945884, 0.9198953859218769, 0.908001147597105, 0.8952913731815082, 0.886953676542665, 0.8771894463102521, 0.8683888934041172, 0.8579399877904575, 0.8482799643655919, 0.8408636339472488, 0.8330870307470335, 0.82434015220157, 0.8181187963300758, 0.8135579418994486, 0.809009026741244 ], "peak_layer": 1, "peak_normalized": 1.3738985779819257, "population_cv": 0.16161423472023242, "rest_mean": 0.0001421073115866252, "spike_contrast": 0.8576417505420824, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012187716347398237, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0001908981939777732, 0.0001847442181315273, 0.00017900991952046752, 0.00017368211410939693, 0.00016855206922627985, 0.00016333843814209104, 0.00015942829486448318, 0.00015560684551019222, 0.0001518549834145233, 0.00014945046859793365, 0.00014607100456487387, 0.0001432897406630218, 0.00014164773165248334, 0.00013921030040364712, 0.000136215501697734, 0.0001340555027127266, 0.0001319019211223349, 0.00012984243221580982, 0.0001278161071240902, 0.00012616344611160457, 0.00012439746933523566, 0.00012323897681199014, 0.00012188227265141904, 0.00012065946793882176, 0.00011920763063244522, 0.00011786540562752634, 0.00011683493357850239, 0.00011575440294109285, 0.0001145390560850501, 0.00011367462138878182, 0.00011304090730845928, 0.00011240885214647278 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 0.00014552221000485588, "first_to_last_ratio": 1.262697055715921, "imbalance_abs_log_ratio": 0.2332499537229619, "last_quartile_mean": 0.00011524712863320019, "mean": 0.00012953460168318998, "normalized": [ 1.1676213631634658, 1.1525055688027706, 1.1391580058081985, 1.1289307982409096, 1.1166731577821778, 1.1054001772932092, 1.094813144753513, 1.0822854389705017, 1.0679308603491235, 1.0564894822224282, 1.0456838360716472, 1.0337072702557388, 1.0234945545465333, 1.0158819477934218, 1.0051772952892828, 0.9951467950359763, 0.9866643397779161, 0.9787559418439278, 0.970448736962688, 0.9607908684400951, 0.9509939230158972, 0.9428061355388626, 0.934924980051271, 0.9261034265480621, 0.9170582603044295, 0.9058562786860701, 0.8981778407016658, 0.8907564927732574, 0.8831168120837269, 0.8778216107141017, 0.8745192645108896, 0.8703053916682415 ], "peak_layer": 1, "peak_normalized": 1.1676213631634658, "population_cv": 0.08967236486753827, "rest_mean": 0.00013110875065817877, "spike_contrast": 0.9231588022206524, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012103419721825049, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00015124736819416285, 0.00014928934979252517, 0.000147560378536582, 0.00014623560127802193, 0.00014464781270362437, 0.00014318757166620344, 0.00014181618462316692, 0.00014019341324456036, 0.0001383339986205101, 0.00013685194426216185, 0.00013545223919209093, 0.00013390085950959474, 0.00013257795944809914, 0.00013159186346456409, 0.00013020524056628346, 0.00012890594371128827, 0.00012780717224813998, 0.00012678276107180864, 0.0001257066905964166, 0.0001244556624442339, 0.00012318661902099848, 0.0001221260172314942, 0.00012110513489460573, 0.0001199624384753406, 0.00011879077646881342, 0.00011733973224181682, 0.00011634510883595794, 0.00011538378748809919, 0.0001143941844929941, 0.00011370827269274741, 0.00011328050459269434, 0.0001127346622524783 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00029741139769612346, "first_to_last_ratio": 2.278072415841521, "imbalance_abs_log_ratio": 0.8233296537390045, "last_quartile_mean": 0.00013055396993877366, "mean": 0.00019328556300024502, "normalized": [ 1.889398576482247, 1.780233894901915, 1.6772594716902607, 1.566224958959574, 1.4762232610465842, 1.375571144634086, 1.3083686481730985, 1.2364401527267181, 1.173148897934763, 1.1508864184295666, 1.080457040307468, 1.0318582947152708, 1.0081957004611959, 0.9698994715426978, 0.9066127339785177, 0.8754550048258108, 0.8446146862114313, 0.8115724449181009, 0.7806677558079268, 0.7613480996239201, 0.7403134026279838, 0.7294052762839086, 0.7160186976339193, 0.7062575707090604, 0.6942777320753531, 0.6866005528973866, 0.6829938476675228, 0.6753403086482458, 0.6715967314167978, 0.6686860556999406, 0.6640286432895989, 0.6600445236791291 ], "peak_layer": 1, "peak_normalized": 1.889398576482247, "population_cv": 0.3579154260008552, "rest_mean": 0.00020340604748559632, "spike_contrast": 0.681567477924495, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013863494677934796, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0003651934675872326, 0.0003440935106482357, 0.00032419004128314555, 0.0003027286729775369, 0.0002853326441254467, 0.00026587804313749075, 0.00025288877077400684, 0.00023898603103589267, 0.00022675274522043765, 0.00022244972933549434, 0.0002088367473334074, 0.00019944331143051386, 0.00019486967357806861, 0.00018746756541077048, 0.00017523515271022916, 0.00016921281348913908, 0.0001632518251426518, 0.00015686523693148047, 0.00015089180669747293, 0.00014715759607497603, 0.00014309189282357693, 0.00014098350948188454, 0.00013839607709087431, 0.0001365093921776861, 0.0001341938623227179, 0.00013270997442305088, 0.00013201285037212074, 0.00013053353177383542, 0.0001298099523410201, 0.00012924736074637622, 0.00012834715016651899, 0.00012757707736454904 ] } } }, "post_attention_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "31853ffb07b24cc59e774bb20656840b4358b3080b42516d380f2dc0f46404c8", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.0022265110164880753, 0.0040087271481752396, 0.005379301495850086, 0.006410556845366955, 0.0027253651060163975, 0.004838948138058186, 0.006175771821290255, 0.00742313964292407, 0.0030937574338167906, 0.005150026176124811, 0.007019025273621082, 0.008143599145114422, 0.004255065228790045, 0.0066224560141563416, 0.008129785768687725, 0.010099578648805618, 0.0041999719105660915, 0.00719039561226964, 0.009030314162373543, 0.010736005380749702, 0.005131533369421959, 0.00787449348717928, 0.009912566281855106, 0.012113703414797783, 0.0059690922498703, 0.00876442901790142, 0.010806876234710217, 0.012353328987956047, 0.006502739619463682, 0.009470963850617409, 0.012123330496251583, 0.01384514570236206 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.00015962236011546338, "first_to_last_ratio": 3.7698422272252903, "imbalance_abs_log_ratio": 1.3270331510418396, "last_quartile_mean": 4.2341920561739244e-05, "mean": 8.61727822893954e-05, "normalized": [ 2.542734798705971, 2.3221448837042376, 2.101145461274622, 1.8911858524935854, 1.7004204056817789, 1.5354142426015858, 1.4229001770142031, 1.3028733810328659, 1.2049412416934622, 1.1705258152816296, 1.0773908232649334, 1.0133220774376726, 0.990609360100461, 0.9272534949358001, 0.8550385909364457, 0.8185651575045715, 0.7703972051466339, 0.7173947009730917, 0.6760256695667757, 0.6514523109028573, 0.618419077603279, 0.6053620388084041, 0.5838930841859182, 0.5697043695032278, 0.5457802700901755, 0.5276149005446632, 0.5104701332743111, 0.49517988582787786, 0.47527562953524777, 0.46579059316028215, 0.45896435764104826, 0.45181000957238127 ], "peak_layer": 1, "peak_normalized": 2.542734798705971, "population_cv": 0.5705202801130907, "rest_mean": 9.280119640472414e-05, "spike_contrast": 0.542873885449774, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.03793460666202e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00021911453222855926, 0.00020010568550787866, 0.00018106155039276928, 0.00016296874673571438, 0.00014652995741926134, 0.00013231091725174338, 0.0001226152671733871, 0.00011227222421439365, 0.00010383313929196447, 0.00010086746624438092, 9.284176485380158e-05, 8.732078276807442e-05, 8.536356472177431e-05, 7.99040135461837e-05, 7.368105434579775e-05, 7.053803710732609e-05, 6.638727063545957e-05, 6.181989738252014e-05, 5.825501284562051e-05, 5.613745815935545e-05, 5.329089253791608e-05, 5.216573117650114e-05, 5.0315691623836756e-05, 4.9093010602518916e-05, 4.703140439232811e-05, 4.546604395727627e-05, 4.398863165988587e-05, 4.267102849553339e-05, 4.095582335139625e-05, 4.013847137684934e-05, 3.955023566959426e-05, 3.893372559105046e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.00021442477373057045, "first_to_last_ratio": 1.8431254562763404, "imbalance_abs_log_ratio": 0.6114627481432037, "last_quartile_mean": 0.00011633759004325839, "mean": 0.00015221248349917005, "normalized": [ 1.7740805953138137, 1.6484457246979003, 1.5301434809496548, 1.4217241600875643, 1.3264958046529274, 1.2454577490392524, 1.1897861330408546, 1.1336265611690741, 1.086914920172498, 1.064301358569379, 1.0227209980571423, 0.9941753882589759, 0.9788979908211966, 0.9511107967643203, 0.9228143274453928, 0.9052447611165039, 0.8846530997418666, 0.8652930871753776, 0.8497381552743479, 0.8388656494639406, 0.8256334792987466, 0.8176309647342147, 0.8076951735882674, 0.8000661307359944, 0.7892605467734257, 0.78144613040264, 0.7731219607566042, 0.7664988488799842, 0.7584935618388989, 0.7529639051546407, 0.7485195296370439, 0.7441790263875565 ], "peak_layer": 1, "peak_normalized": 1.7740805953138137, "population_cv": 0.2736411578809574, "rest_mean": 0.00015762286892384981, "spike_contrast": 0.7803207938393821, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012299640220589936, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00027003721334040165, 0.00025091401766985655, 0.00023290693934541196, 0.0002164041652576998, 0.000201909220777452, 0.0001895742170745507, 0.00018110030214302242, 0.00017255211423616856, 0.00016544201935175806, 0.00016199995297938585, 0.00015567090304102749, 0.00015132590488065034, 0.0001490004942752421, 0.00014477093645837158, 0.00014046386058907956, 0.00013778955326415598, 0.0001346552453469485, 0.00013170840975362808, 0.00012934075493831187, 0.00012768582382705063, 0.00012567172234412283, 0.00012445363972801715, 0.00012294128828216344, 0.00012178005272289738, 0.00012013530795229599, 0.00011894585622940212, 0.00011767881369451061, 0.00011667069338727742, 0.00011545218876563013, 0.00011461050598882139, 0.00011393401655368507, 0.00011327333777444437 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.190154030919075, "first_to_last_ratio": 1.84312545379361, "imbalance_abs_log_ratio": 0.6114627467961817, "last_quartile_mean": 0.10316933691501617, "mean": 0.13498355145566165, "normalized": [ 1.774080628234443, 1.6484456620184311, 1.5301434020504485, 1.4217242444591272, 1.326495766996536, 1.2454576834375484, 1.189786122599316, 1.133626627255238, 1.0869149569673349, 1.0643014004698326, 1.0227209907477024, 0.9941753870542451, 0.9788979589698585, 0.9511107512164827, 0.9228143032197887, 0.9052447972745712, 0.8846531331847195, 0.8652931173486506, 0.8497382179112226, 0.8388656682549475, 0.8256334805754935, 0.8176309131099365, 0.8076951535479161, 0.8000661540388709, 0.7892605573866218, 0.7814461538045975, 0.7731219580171443, 0.7664988542586116, 0.7584935821787124, 0.7529638601652359, 0.7485195164462104, 0.7441789968002049 ], "peak_layer": 1, "peak_normalized": 1.774080628234443, "population_cv": 0.27364115345546325, "rest_mean": 0.13978153549962574, "spike_contrast": 0.7803207857775153, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.10907443761825561, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.2394717037677765, 0.22251304984092712, 0.2065441906452179, 0.19190938770771027, 0.1790551096200943, 0.16811630129814148, 0.16060155630111694, 0.1530209481716156, 0.14671564102172852, 0.1436631828546524, 0.13805051147937775, 0.13419732451438904, 0.13213512301445007, 0.1283843070268631, 0.1245647519826889, 0.12219315767288208, 0.11941362172365189, 0.11680033802986145, 0.11470068246126175, 0.1132330670952797, 0.11144693940877914, 0.11036672443151474, 0.10902556031942368, 0.10799577087163925, 0.10653719305992126, 0.10548237711191177, 0.1043587476015091, 0.10346473753452301, 0.10238415747880936, 0.10163773596286774, 0.10103782266378403, 0.10045192390680313 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 6.883367768750759e-05, "first_to_last_ratio": 2.249759357551583, "imbalance_abs_log_ratio": 0.810823258297238, "last_quartile_mean": 3.059601795030176e-05, "mean": 4.691846504556452e-05, "normalized": [ 1.7359059710084281, 1.6475789727938472, 1.5536456441204052, 1.4921194549590373, 1.4365657321885223, 1.3438513058553838, 1.2787343947807137, 1.2483299149152907, 1.1912354282513808, 1.1603083401306369, 1.099547144002948, 1.059972440022979, 1.0583955432739494, 1.0176800955770247, 0.9754963047746087, 0.9631595708780456, 0.9242400656751482, 0.8752301116137658, 0.8454245377074081, 0.8232853322610043, 0.7863494898171388, 0.7774642190168813, 0.7497486129019063, 0.7388483538626245, 0.709315327139865, 0.6908343794807076, 0.6685469983108738, 0.6592365073812129, 0.6314341746588822, 0.6235898169926835, 0.6199474545603838, 0.6139783610863121 ], "peak_layer": 1, "peak_normalized": 1.7359059710084281, "population_cv": 0.3220406003070692, "rest_mean": 4.9070239684725595e-05, "spike_contrast": 0.7193541792517962, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.52988819940947e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 8.144604362314567e-05, 7.730187644483522e-05, 7.28946688468568e-05, 7.000795449130237e-05, 6.740145909134299e-05, 6.305144052021205e-05, 5.999625500408001e-05, 5.8569723478285596e-05, 5.589093780145049e-05, 5.4439886298496276e-05, 5.158906424185261e-05, 4.973227987647988e-05, 4.9658294301480055e-05, 4.774798799189739e-05, 4.576878927764483e-05, 4.51899686595425e-05, 4.3363925215089694e-05, 4.1064453398576006e-05, 3.9666021621087566e-05, 3.86272840842139e-05, 3.689431105158292e-05, 3.6477427784120664e-05, 3.5177054087398574e-05, 3.466563066467643e-05, 3.3279986382694915e-05, 3.2412888685939834e-05, 3.136719897156581e-05, 3.093036502832547e-05, 2.9625922252307646e-05, 2.9257877031341195e-05, 2.9086982976878062e-05, 2.8806922273361124e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 0.00017341253987979144, "first_to_last_ratio": 1.4955148931299893, "imbalance_abs_log_ratio": 0.4024705576674927, "last_quartile_mean": 0.00011595507385209203, "mean": 0.00013990553861731314, "normalized": [ 1.3768818258178377, 1.3340879911950096, 1.2910054180628048, 1.2527236427111486, 1.2147542166907819, 1.1769889670731195, 1.1474411155803916, 1.1220953663134672, 1.0944075691302455, 1.0773007485615413, 1.0508678613623372, 1.0320596054419082, 1.0208161736522385, 1.000559648637946, 0.9805197815181176, 0.9663128275349431, 0.948744191771126, 0.9319305822463463, 0.9182019814348666, 0.9075038868226692, 0.8943650364968407, 0.886262676120959, 0.8757907045344768, 0.8679002174812178, 0.856288373659771, 0.8478178092769747, 0.8387215581135022, 0.8313975747674917, 0.8224084044020752, 0.8161864847722404, 0.8112200992010112, 0.8064376596145924 ], "peak_layer": 1, "peak_normalized": 1.3768818258178377, "population_cv": 0.16325888721442564, "rest_mean": 0.0001431150389921472, "spike_contrast": 0.8564734877369171, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001225742365932092, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0001926333934534341, 0.00018664629897102714, 0.00018061880837194622, 0.0001752629759721458, 0.00016995084297377616, 0.0001646672753849998, 0.00016053336730692536, 0.00015698735660407692, 0.00015311368042603135, 0.00015072034148033708, 0.00014702223415952176, 0.00014439085498452187, 0.0001428178366040811, 0.00013998383656144142, 0.00013718014815822244, 0.00013519251660909504, 0.00013273456715978682, 0.00013038225006312132, 0.00012846154277212918, 0.00012696482008323073, 0.00012512662215158343, 0.0001239930570591241, 0.00012252797023393214, 0.000121424047392793, 0.00011979948612861335, 0.00011861440725624561, 0.00011734179133782163, 0.0001163171255029738, 0.00011505949078127742, 0.00011418900976423174, 0.00011349418491590768, 0.00011282509512966499 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 0.00014675183047074825, "first_to_last_ratio": 1.2676545313577883, "imbalance_abs_log_ratio": 0.23716836728277432, "last_quartile_mean": 0.00011576642282307148, "mean": 0.00013041823922321782, "normalized": [ 1.1700698937843759, 1.1560440994564782, 1.1417398043557874, 1.130463200949706, 1.118403537277814, 1.1065304334184665, 1.0939881912841023, 1.0846815122144253, 1.0709468270677047, 1.0579907390333845, 1.0442838370166418, 1.035550673160169, 1.025959021745254, 1.0140203095307772, 1.0062047700154308, 0.9976631870190885, 0.9861765922450596, 0.9770016877798172, 0.9687702939454803, 0.9602422119882037, 0.9504995944091904, 0.9425883202531964, 0.9330014667281973, 0.9259385264566372, 0.9147443798098084, 0.906202796813466, 0.8958065504175365, 0.8885655303249116, 0.8806194435723184, 0.8757905905296257, 0.8713783728577705, 0.8681336045391749 ], "peak_layer": 1, "peak_normalized": 1.1700698937843759, "population_cv": 0.09121429133712369, "rest_mean": 0.0001320278307604086, "spike_contrast": 0.921975648780332, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001217264449223876, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0001525984553154558, 0.0001507692359155044, 0.000148903694935143, 0.00014743302017450333, 0.00014586022007279098, 0.00014431175077334046, 0.00014267601363826543, 0.00014146225294098258, 0.000139670999487862, 0.00013798128929920495, 0.00013619365927297622, 0.00013505469541996717, 0.00013380376913119107, 0.00013224674330558628, 0.00013122745440341532, 0.00013011347618885338, 0.00012861541472375393, 0.00012741883983835578, 0.0001263453159481287, 0.0001252330985153094, 0.0001239624834852293, 0.00012293070903979242, 0.00012168040848337114, 0.00012075927224941552, 0.00011929935135412961, 0.00011818537313956767, 0.00011682951299007982, 0.00011588515189941972, 0.00011484883725643158, 0.00011421906674513593, 0.00011364363308530301, 0.00011322045611450449 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00030004180916876066, "first_to_last_ratio": 2.2837506327398005, "imbalance_abs_log_ratio": 0.8258191057718037, "last_quartile_mean": 0.00013138116082700435, "mean": 0.00019473525253488333, "normalized": [ 1.9000096511818072, 1.783700741635724, 1.6708066910504589, 1.570366884717585, 1.4783058799917985, 1.376185321596543, 1.3064973297630273, 1.2402700043336092, 1.1767953849206394, 1.1505341943958838, 1.0771542571420398, 1.026098162637436, 1.013898063759585, 0.9669473025647412, 0.9095398902231254, 0.8755205777387188, 0.842301214161826, 0.8073124727024382, 0.7791991818813049, 0.760574607765185, 0.7402055397447304, 0.7308143413325056, 0.7152667122155245, 0.7043714155678553, 0.6942467749765324, 0.6868749151283189, 0.680655564574882, 0.676363713518914, 0.6701061029149729, 0.6672276319278244, 0.6636177363814285, 0.6582317375530352 ], "peak_layer": 1, "peak_normalized": 1.9000096511818072, "population_cv": 0.35930746026536914, "rest_mean": 0.00020494150898971216, "spike_contrast": 0.6812747128050888, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013962146767880766, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0003699988592416048, 0.0003473494143690914, 0.0003253649629186839, 0.0003058057918678969, 0.0002878782688640058, 0.0002679917961359024, 0.0002544210874475539, 0.00024152429250534624, 0.00022916354646440595, 0.000224049566895701, 0.00020975990628357977, 0.00019981748482678086, 0.000197441695490852, 0.0001882987271528691, 0.00017711948021315038, 0.00017049472080543637, 0.00016402573965024203, 0.00015721219824627042, 0.00015173754945863038, 0.00014811068831477314, 0.00014414411270990968, 0.0001423153153154999, 0.0001392876438330859, 0.00013716594548895955, 0.00013519432104658335, 0.00013375876005738974, 0.00013254763325676322, 0.0001317118585575372, 0.00013049328117631376, 0.00012993274140171707, 0.00012922976748086512, 0.00012818092363886535 ] } } }, "post_mlp_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "7fd9c911e93175896ad618842c120e77b3f017c17b7eec9c2bfb01fc6dbc8ac2", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.0034885562490671873, 0.004831995815038681, 0.005952076520770788, 0.006925336085259914, 0.00385080324485898, 0.005542940925806761, 0.006729288958013058, 0.007927131839096546, 0.004081308841705322, 0.005811207927763462, 0.007534808944910765, 0.008581922389566898, 0.00499920267611742, 0.007102519739419222, 0.008542952127754688, 0.010447698645293713, 0.005041283555328846, 0.007636935915797949, 0.009387043304741383, 0.011020516976714134, 0.005807018838822842, 0.008367395959794521, 0.01024632342159748, 0.012378672137856483, 0.006453148555010557, 0.009127611294388771, 0.011037957854568958, 0.012611051090061665, 0.007010402157902718, 0.009794782847166061, 0.012445803731679916, 0.014087481424212456 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.00015858902861509705, "first_to_last_ratio": 3.757869257587208, "imbalance_abs_log_ratio": 1.3238521099622325, "last_quartile_mean": 4.220184837322449e-05, "mean": 8.57325238712292e-05, "normalized": [ 2.5462257183789148, 2.3032637538343064, 2.0924289533379263, 1.8886492391629932, 1.7031256747388024, 1.5374176267930122, 1.4224231993460965, 1.3049597038569594, 1.2055618292025094, 1.169245405308266, 1.0769812048488787, 1.0144334047898542, 0.9913512297333369, 0.9317090592952362, 0.8577499774884012, 0.81555804918997, 0.7682725064221458, 0.7217807351250844, 0.6783324192661732, 0.6511800118936032, 0.6192916242842709, 0.6053728292385717, 0.5848720876072855, 0.5718124973482859, 0.5462302773879167, 0.5275173250132701, 0.5112998770983732, 0.49691833866078555, 0.47721658788927634, 0.46551989467389665, 0.46012635628943105, 0.45317260249616614 ], "peak_layer": 1, "peak_normalized": 2.5462257183789148, "population_cv": 0.5686111305901927, "rest_mean": 9.231303704803361e-05, "spike_contrast": 0.5437775023084314, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.019775271648541e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00021829435718245804, 0.00019746461475733668, 0.0001793892151908949, 0.0001619186659809202, 0.00014601326256524771, 0.00013180669338908046, 0.00012194793089292943, 0.00011187748896190897, 0.00010335585830034688, 0.00010024235962191597, 9.233231685357168e-05, 8.69699360919185e-05, 8.499104296788573e-05, 7.987776916706935e-05, 7.353707042057067e-05, 6.991984992055222e-05, 6.586594099644572e-05, 6.188008410390466e-05, 5.8155150327365845e-05, 5.582730591413565e-05, 5.309343396220356e-05, 5.190014053368941e-05, 5.014256021240726e-05, 4.90229285787791e-05, 4.682970029534772e-05, 4.522539165918715e-05, 4.383502891869284e-05, 4.2602063331287354e-05, 4.0912982512963936e-05, 3.991019548266195e-05, 3.944779382436536e-05, 3.885163096128963e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.00021242368711682502, "first_to_last_ratio": 1.8345658968465297, "imbalance_abs_log_ratio": 0.6068078850437921, "last_quartile_mean": 0.0001157896194854402, "mean": 0.0001511171253696375, "normalized": [ 1.7717451736709768, 1.636674793183293, 1.5242755762025908, 1.4186515254985965, 1.3264266232459345, 1.2453738409548603, 1.1893512719369177, 1.1330135273991528, 1.085860457802012, 1.0628542818802846, 1.023252039669589, 0.9942070699734702, 0.9784308644775869, 0.9531083940389613, 0.9232390376562555, 0.9040072643238835, 0.8853775311228679, 0.8680181295617843, 0.8515367513400655, 0.8395795332993111, 0.8267439067367663, 0.8185133286645813, 0.8091730400239778, 0.800791325997512, 0.7909899739933552, 0.7821072339266707, 0.7753403490721149, 0.7682422073314642, 0.7604971517068638, 0.7549863467424895, 0.7508596466770359, 0.7467718018887739 ], "peak_layer": 1, "peak_normalized": 1.7717451736709768, "population_cv": 0.2714186609524326, "rest_mean": 0.00015645541295978345, "spike_contrast": 0.7816308178118705, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012229037238284946, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0002677410375326872, 0.0002473295899108052, 0.00023034414334688336, 0.00021438254043459892, 0.00020044577831868082, 0.00018819731485564262, 0.00017973134526982903, 0.000171217747265473, 0.0001640921109355986, 0.00016061548376455903, 0.00015463090676348656, 0.00015024171443656087, 0.0001478576596127823, 0.00014403100067283958, 0.00013951722939964384, 0.00013661097909789532, 0.00013379570737015456, 0.0001311724045081064, 0.0001286817860091105, 0.00012687484559137374, 0.0001249351626029238, 0.00012369138130452484, 0.00012227990373503417, 0.00012101328320568427, 0.00011953213106608018, 0.0001181897969217971, 0.0001171672047348693, 0.00011609455395955592, 0.00011492414341773838, 0.0001140913664130494, 0.00011346775136189535, 0.00011285000800853595 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.18837944976985455, "first_to_last_ratio": 1.8345658800697515, "imbalance_abs_log_ratio": 0.6068078758989703, "last_quartile_mean": 0.10268339328467846, "mean": 0.1340121782850474, "normalized": [ 1.7717451751532867, 1.6366747548098712, 1.5242756140248972, 1.4186514377480737, 1.3264266357207033, 1.2453739091726768, 1.1893513017871604, 1.1330134634090196, 1.0858604665419038, 1.0628542749024097, 1.0232520332438115, 0.9942070832344971, 0.9784308561813967, 0.9531084116998004, 0.9232390798282929, 0.9040072094634526, 0.8853775581660035, 0.8680181154203164, 0.8515367049614931, 0.8395794986042229, 0.8267439263823189, 0.8185133396785964, 0.809173050926764, 0.8007913534932437, 0.7909900046670136, 0.782107217826699, 0.7753403703776178, 0.7682422246066585, 0.7604971603471317, 0.7549863443945886, 0.7508596538022533, 0.7467717694238248 ], "peak_layer": 1, "peak_normalized": 1.7717451751532867, "population_cv": 0.27141865758333467, "rest_mean": 0.13874622461972413, "spike_contrast": 0.7816308398662988, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.10844832807779312, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.23743543028831482, 0.21933434903621674, 0.20427149534225464, 0.19011656939983368, 0.17775732278823853, 0.16689527034759521, 0.15938755869865417, 0.15183760225772858, 0.14551852643489838, 0.14243541657924652, 0.13712823390960693, 0.1332358568906784, 0.1311216503381729, 0.12772813439369202, 0.1237252801656723, 0.12114797532558441, 0.1186513751745224, 0.1163249984383583, 0.11411628872156143, 0.11251387745141983, 0.11079375445842743, 0.10969075560569763, 0.10843904316425323, 0.107315793633461, 0.10600229352712631, 0.104811891913414, 0.10390505194664001, 0.10295381397008896, 0.1019158810377121, 0.10117736458778381, 0.10062433779239655, 0.10007651150226593 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 6.873207303215167e-05, "first_to_last_ratio": 2.248757814140914, "imbalance_abs_log_ratio": 0.8103779811585758, "last_quartile_mean": 3.0564462122129044e-05, "mean": 4.6857780716891284e-05, "normalized": [ 1.7400119939228862, 1.6405409364325143, 1.555121247708426, 1.4944054279879166, 1.4327475398601994, 1.3414100714975594, 1.2804101716466747, 1.2499370459040815, 1.192804641985012, 1.1608858830017554, 1.098191983597808, 1.057744066133373, 1.0542717513613309, 1.021869458338179, 0.9775749297615711, 0.9570927445976433, 0.9210091333285576, 0.8815125232807631, 0.847820728512489, 0.8204970180441978, 0.7871838673851995, 0.7759041238100245, 0.7506938949785397, 0.7421070509174572, 0.7072987335050381, 0.6891811912847667, 0.6695407652827761, 0.6616333377893114, 0.6345027924930273, 0.6212629426015038, 0.6200302724102036, 0.614801730639213 ], "peak_layer": 1, "peak_normalized": 1.7400119939228862, "population_cv": 0.32175451551289735, "rest_mean": 4.9004235565748616e-05, "spike_contrast": 0.7196709452950106, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.5266924533061685e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 8.153310045599937e-05, 7.687210745643824e-05, 7.28695304132998e-05, 7.002452184678987e-05, 6.713537004543468e-05, 6.28554989816621e-05, 5.9997179050697014e-05, 5.8569276006892323e-05, 5.589217835222371e-05, 5.439653614303097e-05, 5.1458839152473956e-05, 4.9563539505470544e-05, 4.9400834541302174e-05, 4.788253500009887e-05, 4.58069916930981e-05, 4.4847241952084005e-05, 4.315644400776364e-05, 4.130572051508352e-05, 3.972699778387323e-05, 3.8446669350378215e-05, 3.688568904181011e-05, 3.635714529082179e-05, 3.5175849916413426e-05, 3.477348946034908e-05, 3.3142448955914006e-05, 3.2293501135427505e-05, 3.13731943606399e-05, 3.1002669857116416e-05, 2.9731392714893445e-05, 2.9111002731951885e-05, 2.905324254243169e-05, 2.8808244678657502e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 0.00017190765356644988, "first_to_last_ratio": 1.4894621368061003, "imbalance_abs_log_ratio": 0.3984150727783497, "last_quartile_mean": 0.00011541592721187044, "mean": 0.00013894661265112518, "normalized": [ 1.3739040544627474, 1.3296102750467884, 1.2883342981472405, 1.2499918159027226, 1.2130627732820631, 1.175544413473833, 1.1474083142393565, 1.1199112790671786, 1.092909618037032, 1.0755929927697745, 1.0512861461262244, 1.0312571117882559, 1.0194388253067554, 1.0019061415077835, 0.980344379776871, 0.9647961249933756, 0.9493075664595542, 0.9344817410273759, 0.9198929201794295, 0.9079992279088673, 0.8952920956888009, 0.8869530532606651, 0.8771902552778059, 0.8683889858357474, 0.8579407806493952, 0.8482823463732029, 0.8408658204630546, 0.8330843095877946, 0.8243382853592501, 0.818115422616924, 0.8135611224181157, 0.8090075029660142 ], "peak_layer": 1, "peak_normalized": 1.3739040544627474, "population_cv": 0.16161403917150835, "rest_mean": 0.00014210757001786045, "spike_contrast": 0.8576421569620596, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012187744287075474, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00019089931447524577, 0.00018474484386388212, 0.00017900968668982387, 0.00017368212866131216, 0.0001685509632807225, 0.00016333791427314281, 0.00015942849859129637, 0.00015560787869617343, 0.00015185608936008066, 0.00014945000293664634, 0.00014607264893129468, 0.0001432896824553609, 0.00014164757158141583, 0.0001392114645568654, 0.00013621553080156446, 0.0001340551534667611, 0.00013190307072363794, 0.00012984307250007987, 0.0001278160052606836, 0.00012616341700777411, 0.0001243978040292859, 0.00012323912233114243, 0.00012188261462142691, 0.00012065970804542303, 0.00011920796532649547, 0.00011786595860030502, 0.00011683545744745061, 0.00011575424287002534, 0.00011453901242930442, 0.00011367436673026532, 0.00011304156214464456, 0.00011240885214647278 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 0.00014552358334185556, "first_to_last_ratio": 1.262638843036328, "imbalance_abs_log_ratio": 0.2332038508028458, "last_quartile_mean": 0.00011525352965691127, "mean": 0.00012953498708156985, "normalized": [ 1.1676304712442283, 1.1525836984023852, 1.1391521450612143, 1.1288261090295841, 1.1167600441451297, 1.1054423860149858, 1.094731137320412, 1.082319740349084, 1.067957453000188, 1.0563935463552665, 1.04560972624621, 1.0337959762176818, 1.0235535208863231, 1.0159040893770892, 1.0050628637064216, 0.9950792389307415, 0.9865814183778941, 0.9786230528286792, 0.9704418054120744, 0.9608381133357455, 0.9508407831226554, 0.9429749854457838, 0.9346839821902475, 0.9262276149685831, 0.9170542960948607, 0.9059488475652709, 0.8984648923618161, 0.8908037775288885, 0.8831483920181569, 0.8778579808382188, 0.8744051093280881, 0.8703028022960919 ], "peak_layer": 1, "peak_normalized": 1.1676304712442283, "population_cv": 0.0896642712448568, "rest_mean": 0.00013110964479252557, "spike_contrast": 0.9231344927669946, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012103183544240892, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00015124899800866842, 0.00014929991448298097, 0.00014756005839444697, 0.00014622247545048594, 0.00014465949789155275, 0.00014319346519187093, 0.00014180598373059183, 0.00014019827358424664, 0.00013833785487804562, 0.00013683992438018322, 0.0001354430423816666, 0.00013391274842433631, 0.0001325859921053052, 0.00013159512309357524, 0.00013019080506637692, 0.00012889757636003196, 0.00012779681128449738, 0.00012676592450588942, 0.00012570616672746837, 0.00012446215259842575, 0.00012316714855842292, 0.00012214825255796313, 0.00012107427755836397, 0.00011997888213954866, 0.00011879061639774591, 0.00011735207226593047, 0.0001163826382253319, 0.0001153902558144182, 0.00011439861555118114, 0.00011371332220733166, 0.00011326605454087257, 0.0001127346622524783 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00029742828155576717, "first_to_last_ratio": 2.2781890121807993, "imbalance_abs_log_ratio": 0.8233808344454435, "last_quartile_mean": 0.00013055469935352448, "mean": 0.00019329176166138495, "normalized": [ 1.8891850069388627, 1.7801603926271052, 1.6773544463608039, 1.566500262900107, 1.476456430986117, 1.3754925511314788, 1.3084504582544147, 1.236424592515421, 1.172965600373877, 1.150871569148191, 1.0802294364746856, 1.0319209662753672, 1.0080063247818516, 0.9698985570693867, 0.9067931019504507, 0.8756163462587222, 0.8444435806568995, 0.8113939671049883, 0.7809677247391495, 0.7613442367276676, 0.7402582678385915, 0.7293562882295931, 0.7162426695419845, 0.7062419232587185, 0.6940340549797547, 0.6866100033591334, 0.6831688143117768, 0.6753320518823956, 0.6715919825156708, 0.6686410475337363, 0.6640239865195754, 0.6600233567535221 ], "peak_layer": 1, "peak_normalized": 1.8891850069388627, "population_cv": 0.357915029863326, "rest_mean": 0.00020341356920888993, "spike_contrast": 0.6815376252628051, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001386340009048581, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00036516389809548855, 0.0003440903383307159, 0.0003242187958676368, 0.0003027915954589844, 0.00028538686456158757, 0.00026587137836031616, 0.0002529126941226423, 0.00023899068764876574, 0.00022672458726447076, 0.00022245399304665625, 0.0002087994507746771, 0.00019946182146668434, 0.00019483931828290224, 0.00018747340072877705, 0.00017527563613839447, 0.00016924942610785365, 0.00016322398732881993, 0.00015683576930314302, 0.0001509546273155138, 0.00014716156874783337, 0.0001430858246749267, 0.0001409785618307069, 0.00013844380737282336, 0.00013651074550580233, 0.00013415106514003128, 0.00013271605712361634, 0.00013205090363044292, 0.00013053612201474607, 0.00012981319741811603, 0.00012924280599690974, 0.00012835036613978446, 0.00012757707736454904 ] } } }, "pre_attention_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "de71d4cf2d509a1063f6eac7f50c4885e9a058dee7e4b993ad40f5527f528d00", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.02802751213312149, 0.01413465291261673, 0.014225807972252369, 0.01433455292135477, 0.01440478302538395, 0.009703479707241058, 0.00977881159633398, 0.009858808480203152, 0.009950309991836548, 0.00752287870272994, 0.007571155205368996, 0.007693085819482803, 0.007788818795233965, 0.006353048607707024, 0.006389216054230928, 0.0064707002602517605, 0.00656962301582098, 0.005553483963012695, 0.0056249531917274, 0.005688921548426151, 0.005768724251538515, 0.004966245498508215, 0.005057469941675663, 0.005098348017781973, 0.005216081161051989, 0.0046758754178881645, 0.004727107938379049, 0.004777040798217058, 0.004837613552808762, 0.0044186655431985855, 0.004510571248829365, 0.0045849340967834 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.0001580166108396952, "first_to_last_ratio": 1.9713740353643698, "imbalance_abs_log_ratio": 0.6787307795042399, "last_quartile_mean": 8.015557068574708e-05, "mean": 0.00011843875881822896, "normalized": [ 0.8918679597819287, 1.5814819593588525, 1.5722239987443252, 1.3246067564285042, 1.1252650527356378, 1.512743589128775, 1.3788112428745087, 1.2863035124436846, 1.097095055843533, 1.468154353173571, 1.1523254606068964, 1.0839888571909224, 1.054518217028451, 1.146335080549274, 1.1665208527784754, 0.8931136212023528, 0.8203804970547371, 1.073741223277434, 0.919654742394771, 0.8223971330190447, 0.7711288002244273, 0.8470775332584999, 0.8298711629738247, 0.7662487463744663, 0.7210382462310316, 0.7071376059209309, 0.7189559392079661, 0.727980518763353, 0.6111916361855597, 0.6986829946379066, 0.6244678178312321, 0.6046898327751223 ], "peak_layer": 2, "peak_normalized": 1.5814819593588525, "population_cv": 0.2904715201464359, "rest_mean": 0.00012310891098523926, "spike_contrast": 0.7572151875143335, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.321993711637333e-05, "top_five_layers": [ 2, 3, 6, 10, 7 ], "top_five_spike_overlap": 0 }, "values": [ 0.00010563173418631777, 0.0001873087603598833, 0.00018621225899551064, 0.00015688478015363216, 0.0001332749961875379, 0.00017916747310664505, 0.00016330469225067645, 0.00015234819147735834, 0.00012993857671972364, 0.00017388637934345752, 0.0001364799973089248, 0.0001283862948184833, 0.00012489582877606153, 0.00013577050413005054, 0.00013816128193866462, 0.00010577926877886057, 9.71648478298448e-05, 0.00012717257777694613, 0.00010892276623053476, 9.74036956904456e-05, 9.133153798757121e-05, 0.0001003268116619438, 9.828891052166e-05, 9.07535504666157e-05, 8.539887494407594e-05, 8.375250035896897e-05, 8.515224908478558e-05, 8.622110908618197e-05, 7.238877878990024e-05, 8.275114669231698e-05, 7.396119326585904e-05, 7.161871326388791e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.0001664285200604354, "first_to_last_ratio": 1.9719337364651075, "imbalance_abs_log_ratio": 0.6790146534176558, "last_quartile_mean": 8.439863722742302e-05, "mean": 0.0001245867656507471, "normalized": [ 0.8984334461398115, 1.5927952483854981, 1.5658595485525297, 1.3198735865176374, 1.131213894321221, 1.5125315092450449, 1.3796810696180641, 1.286366052942558, 1.0960606282761096, 1.4536884557404346, 1.1543342746982923, 1.0797559630746076, 1.0497530570909623, 1.1457182987306291, 1.1551962694199982, 0.9023721663305979, 0.8238710779144349, 1.0632503993742801, 0.9211624832492573, 0.8274335806257342, 0.7739032412019898, 0.850990856837025, 0.8281317613098801, 0.7681943949939335, 0.7208251570035091, 0.7089546837555952, 0.7203162530736348, 0.7236174125342815, 0.6145740290254166, 0.6994991973458914, 0.6274381906330746, 0.6042038120380651 ], "peak_layer": 2, "peak_normalized": 1.5927952483854981, "population_cv": 0.2889665774838781, "rest_mean": 0.00012946851159169995, "spike_contrast": 0.7586812913967177, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.822533756960183e-05, "top_five_layers": [ 2, 3, 6, 10, 7 ], "top_five_spike_overlap": 0 }, "values": [ 0.00011193291720701382, 0.0001984412083402276, 0.0001950853766174987, 0.000164438781212084, 0.00014093428035266697, 0.00018844140868168324, 0.00017189000209327787, 0.00016026418597903103, 0.00013655464863404632, 0.00018111034296453, 0.00014381477376446128, 0.0001345233031315729, 0.00013078533811494708, 0.00014274133718572557, 0.00014392216689884663, 0.00011242362961638719, 0.00010264343291055411, 0.0001324669283349067, 0.00011476465442683548, 0.00010308727360097691, 9.641810174798593e-05, 0.00010602219845168293, 0.00010317425767425448, 9.570685506332666e-05, 8.980527491075918e-05, 8.832637104205787e-05, 8.974187221610919e-05, 9.015315299620852e-05, 7.656779052922502e-05, 8.714834257261828e-05, 7.817049481673166e-05, 7.527579873567447e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.14759047608822584, "first_to_last_ratio": 1.9719337947467823, "imbalance_abs_log_ratio": 0.6790146829732503, "last_quartile_mean": 0.07484555337578058, "mean": 0.1104847895912826, "normalized": [ 0.8984334974418968, 1.5927952138130326, 1.565859646322909, 1.3198735575362357, 1.1312139236939245, 1.5125315007911495, 1.3796810264894532, 1.286366010451999, 1.0960606168617568, 1.4536884922319049, 1.1543342013148132, 1.0797558953237962, 1.0497530983161607, 1.1457182562234747, 1.1551962948455465, 0.9023721940039697, 0.8238711061937815, 1.0632504861804335, 0.9211625151546727, 0.8274335809310764, 0.7739032642161865, 0.8509909070635955, 0.8281317391228538, 0.7681943896830757, 0.7208251689669645, 0.7089546589522648, 0.7203162343708721, 0.7236173971219663, 0.6145740294939214, 0.6994991437375198, 0.6274381319743458, 0.6042038211744466 ], "peak_layer": 2, "peak_normalized": 1.5927952138130326, "population_cv": 0.28896658018674404, "rest_mean": 0.11481397047086998, "spike_contrast": 0.7586813040631772, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.08710721284151077, "top_five_layers": [ 2, 3, 6, 10, 7 ], "top_five_spike_overlap": 0 }, "values": [ 0.09926323592662811, 0.1759796440601349, 0.1730036735534668, 0.14582595229148865, 0.12498193234205246, 0.16711172461509705, 0.15243376791477203, 0.14212387800216675, 0.12109802663326263, 0.16061046719551086, 0.1275363713502884, 0.1192966029047966, 0.11598175019025803, 0.12658444046974182, 0.12763161957263947, 0.09969840198755264, 0.0910252258181572, 0.11747300624847412, 0.10177444666624069, 0.09141882508993149, 0.08550453931093216, 0.09402155131101608, 0.09149596095085144, 0.08487379550933838, 0.0796402171254158, 0.0783287063241005, 0.07958398759365082, 0.07994871586561203, 0.06790108233690262, 0.07728401571512222, 0.069322369992733, 0.06675533205270767 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 5.093040158499207e-05, "first_to_last_ratio": 0.8860093039525244, "imbalance_abs_log_ratio": 0.12102782735710801, "last_quartile_mean": 5.748291960117058e-05, "mean": 5.7093861016710434e-05, "normalized": [ 0.4337136137178416, 0.9185569667519601, 0.9444095926066538, 0.8728787927927439, 0.7849140301431445, 1.1296292591692207, 1.0726593944235638, 0.9796135302182263, 0.9386642177229508, 1.3046353648934683, 1.0461832118506482, 1.0032917361864164, 0.9662905330123907, 1.0760080966276753, 1.1828739682986207, 0.9087955578115051, 0.9194941496879511, 1.198292373503509, 1.0946132296674007, 1.0673695485587213, 0.9244260207272295, 1.0525679458300632, 1.125653431968099, 0.9999504891362015, 0.9629027708994021, 1.0063234987710272, 0.9975933861195996, 1.0791525148711678, 0.9412468232151083, 1.085173475571988, 1.0031556955576197, 0.9789667796878816 ], "peak_layer": 10, "peak_normalized": 1.3046353648934683, "population_cv": 0.1441687540611157, "rest_mean": 5.695535414647397e-05, "spike_contrast": 1.0155638391297448, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.784179811598733e-05, "top_five_layers": [ 10, 18, 15, 6, 23 ], "top_five_spike_overlap": 1 }, "values": [ 2.4762384782661684e-05, 5.2443963795667514e-05, 5.3919990023132414e-05, 4.9836020480142906e-05, 4.481377254705876e-05, 6.449489592341706e-05, 6.124226638348773e-05, 5.592991874436848e-05, 5.3591964388033375e-05, 7.448667020071298e-05, 5.973063889541663e-05, 5.728179894504137e-05, 5.516925739357248e-05, 6.143345672171563e-05, 6.75348419463262e-05, 5.1886647270293906e-05, 5.249747118796222e-05, 6.84151382301934e-05, 6.249569560168311e-05, 6.094024865888059e-05, 5.277905074763112e-05, 6.009516800986603e-05, 6.426790059776977e-05, 5.7091034250333905e-05, 5.4975836974335834e-05, 5.74548939766828e-05, 5.695645813830197e-05, 6.161298369988799e-05, 5.373941530706361e-05, 6.19567435933277e-05, 5.727403186028823e-05, 5.58929932594765e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 5.801741053801379e-05, "first_to_last_ratio": 0.954195857595535, "imbalance_abs_log_ratio": 0.046886327142486896, "last_quartile_mean": 6.08024129178375e-05, "mean": 6.296396202287724e-05, "normalized": [ 0.4796364558374551, 0.9568353419130547, 0.9804638447804884, 0.8949545354119591, 0.8237377234766136, 1.1597846916175047, 1.085649359633547, 0.9904450076993521, 0.9595078978373608, 1.2941748206155441, 1.0680753760655664, 1.0231559828609242, 0.9904019047543832, 1.096490963646882, 1.1850351594596897, 0.9422215946487361, 0.9418631353047864, 1.1705024179981143, 1.0926353878857977, 1.0382275689133567, 0.9320833313178482, 1.0784583323883348, 1.0985080196909318, 0.9917906747078287, 0.9476388143731139, 0.9855066582268529, 0.978396290111904, 1.028407839812513, 0.8982055143685478, 1.0282804954549911, 0.9451124964564701, 0.9138123627295472 ], "peak_layer": 10, "peak_normalized": 1.2941748206155441, "population_cv": 0.13431233496292436, "rest_mean": 6.285090866084729e-05, "spike_contrast": 1.0115120295379672, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.357445017783903e-05, "top_five_layers": [ 10, 15, 18, 6, 23 ], "top_five_spike_overlap": 1 }, "values": [ 3.019981159013696e-05, 6.0246144130360335e-05, 6.173388828756288e-05, 5.634988337988034e-05, 5.186579073779285e-05, 7.302463927771896e-05, 6.835678505012766e-05, 6.236234185053036e-05, 6.041441884008236e-05, 8.148637425620109e-05, 6.725025741616264e-05, 6.442195444833487e-05, 6.235962791834027e-05, 6.903941539349034e-05, 7.461450877599418e-05, 5.9326004702597857e-05, 5.930343468207866e-05, 7.369946979451925e-05, 6.879665306769311e-05, 6.537092122016475e-05, 5.8687659475253895e-05, 6.790400948375463e-05, 6.916641723364592e-05, 6.244707037694752e-05, 5.966709431959316e-05, 6.205140380188823e-05, 6.160370685392991e-05, 6.475263216998428e-05, 5.655457789544016e-05, 6.474461406469345e-05, 5.950802733423188e-05, 5.753724690293893e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 1.7888660295284353e-05, "first_to_last_ratio": 0.41855462748958633, "imbalance_abs_log_ratio": 0.8709478659353102, "last_quartile_mean": 4.2739129185065394e-05, "mean": 3.1986895351110434e-05, "normalized": [ 0.2593955501951102, 0.4997829486650695, 0.48480994473028793, 0.4968760363401999, 0.5474125689219601, 0.7788406177150305, 0.7554473604850604, 0.6514322393717875, 0.7578135841604356, 0.9674810188795085, 0.9032772556875033, 0.9019220651946442, 0.8311356963419073, 0.9587970177975769, 1.1401394869948454, 0.8953886492354074, 1.0262272842074007, 1.1413916918814533, 1.174037958462901, 1.194927715786966, 1.0267140632187253, 1.3434898267078448, 1.3729067467361273, 1.2011929483563188, 1.1735959904166378, 1.3266745689353452, 1.2739952988364465, 1.4049535244931144, 1.2752889026856997, 1.4680979746684197, 1.4182292846928506, 1.3483241791974143 ], "peak_layer": 30, "peak_normalized": 1.4680979746684197, "population_cv": 0.3184955307507873, "rest_mean": 3.066251991657299e-05, "spike_contrast": 1.276428773763581, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.913852269761264e-05, "top_five_layers": [ 30, 31, 28, 23, 32 ], "top_five_spike_overlap": 1 }, "values": [ 8.297258318634704e-06, 1.5986504877218977e-05, 1.5507564967265353e-05, 1.589352177688852e-05, 1.7510028555989265e-05, 2.491269333404489e-05, 2.416441566310823e-05, 2.083729486912489e-05, 2.4240103812189773e-05, 3.094671410508454e-05, 2.889303505071439e-05, 2.8849686714238487e-05, 2.6585450541460887e-05, 3.066893987124786e-05, 3.6469522456172854e-05, 2.8640703021665104e-05, 3.282582474639639e-05, 3.6509576602838933e-05, 3.755382931558415e-05, 3.822202779701911e-05, 3.2841395295690745e-05, 4.2974068492185324e-05, 4.391502443468198e-05, 3.842243313556537e-05, 3.7539692129939795e-05, 4.243620060151443e-05, 4.0751154301688075e-05, 4.494010136113502e-05, 4.0792532672639936e-05, 4.695989628089592e-05, 4.5364751713350415e-05, 4.3128704419359565e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00022091567916504573, "first_to_last_ratio": 1.232986382535498, "imbalance_abs_log_ratio": 0.20943917994887448, "last_quartile_mean": 0.00017917122386279516, "mean": 0.0002198539991695725, "normalized": [ 0.4672441326905094, 1.0299890566632133, 1.08986441571188, 1.0084939820045402, 0.8814237539069011, 1.2693826047962031, 1.1886514905905403, 1.1035827467935222, 1.0473307057033197, 1.4274044616309502, 1.1442998336482333, 1.0911283607583675, 1.1094038041384326, 1.1946873974128753, 1.2511466100570947, 0.9923165311956169, 0.9239703707694655, 1.2473578196113966, 1.1181336030529359, 1.0160745414007064, 0.9397394354798955, 0.9905446518023159, 1.0121913654482901, 0.9359943959598221, 0.8839446941554127, 0.8510229499852975, 0.8648513533654478, 0.8786898836618169, 0.7455559493639876, 0.8399702482950041, 0.7392957942269532, 0.7163130557190521 ], "peak_layer": 10, "peak_normalized": 1.4274044616309502, "population_cv": 0.18952709251819305, "rest_mean": 0.0002217885959436427, "spike_contrast": 0.9441746799407325, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00020940717658959329, "top_five_layers": [ 10, 6, 15, 18, 14 ], "top_five_spike_overlap": 0 }, "values": [ 0.00010272549116052687, 0.00022644721320830286, 0.00023961105034686625, 0.00022172143508214504, 0.0001937845372594893, 0.0002790788421407342, 0.0002613297838252038, 0.00024262708029709756, 0.00023025984410196543, 0.000313820579322055, 0.00025157889467664063, 0.00023988893372006714, 0.00024390686303377151, 0.000262656802078709, 0.00027506958576850593, 0.0002181647578254342, 0.00020313858112785965, 0.00027423660503700376, 0.0002458261442370713, 0.00022338805138133466, 0.00020660547306761146, 0.00021777520305477083, 0.00022253431961871684, 0.00020578211115207523, 0.0001943387760547921, 0.00018710079893935472, 0.0001901410287246108, 0.00019318348495289683, 0.00016391345707233995, 0.00018467081827111542, 0.00016253713693004102, 0.00015748428995721042 ] } } }, "pre_mlp_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "2999a257265024350fd2665e2a8d452a03cd662193b4245d15290408ad29922d", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.014061377383768559, 0.014165290631353855, 0.014281988143920898, 0.014350161887705326, 0.00966146495193243, 0.009741888381540775, 0.009820535778999329, 0.00991273857653141, 0.0074927168898284435, 0.007546746172010899, 0.00766373984515667, 0.007762390188872814, 0.006325984839349985, 0.006375366821885109, 0.006445344537496567, 0.006555322092026472, 0.005531130358576775, 0.005611324217170477, 0.0056777093559503555, 0.00574630219489336, 0.004946626722812653, 0.0050392793491482735, 0.005084683652967215, 0.005203457549214363, 0.004667261149734259, 0.004717131145298481, 0.004768412560224533, 0.004826710093766451, 0.0044052014127373695, 0.004504534415900707, 0.004574235528707504, 0.004644096828997135 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 3.729895206561196e-05, "first_to_last_ratio": 1.2469721805352876, "imbalance_abs_log_ratio": 0.22071835733540474, "last_quartile_mean": 2.9911615229138988e-05, "mean": 3.340667609563752e-05, "normalized": [ 1.308752778077389, 1.2456464596610868, 1.0236956931411572, 0.9922285647386262, 1.26903181062375, 1.1170846373362797, 1.0287316533628064, 0.9469237624012169, 1.1300261751959062, 1.1132698789554223, 1.0126240729566007, 0.9379202574439188, 1.1130503370514597, 1.070713488608195, 0.9897642722064988, 0.922085688720865, 1.036831181462568, 0.9898105546019274, 0.9395130252451869, 0.9001967292772246, 0.9975952535130206, 0.9404117749145336, 0.9224862220158919, 0.8885787590271026, 0.9461670180110204, 0.8974372095121397, 0.8904691498559152, 0.8684437934704528, 0.9244147277585869, 0.9015689206268677, 0.8740901912628669, 0.8604359589635168 ], "peak_layer": 1, "peak_normalized": 1.308752778077389, "population_cv": 0.1164905978879517, "rest_mean": 3.378375206213807e-05, "spike_contrast": 0.9285666618330386, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.137046587653458e-05, "top_five_layers": [ 1, 5, 2, 9, 6 ], "top_five_spike_overlap": 0 }, "values": [ 4.3721080146497115e-05, 4.161290780757554e-05, 3.419827044126578e-05, 3.314705827506259e-05, 4.239413465256803e-05, 3.73180846509058e-05, 3.4366505133220926e-05, 3.1633575417799875e-05, 3.7750418414361775e-05, 3.719064625329338e-05, 3.3828404411906376e-05, 3.1332798243965954e-05, 3.718331208801828e-05, 3.5768978705164045e-05, 3.306473445263691e-05, 3.080381793552078e-05, 3.463708344497718e-05, 3.3066280593629926e-05, 3.138600732199848e-05, 3.007258055731654e-05, 3.332634150865488e-05, 3.1416031561093405e-05, 3.0817198421573266e-05, 2.9684462788281962e-05, 3.1608295103069395e-05, 2.9980394174344838e-05, 2.9747614462394267e-05, 2.9011820515734144e-05, 3.0881623388268054e-05, 3.01184209092753e-05, 2.9200447897892445e-05, 2.874430538213346e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 6.705871965095866e-05, "first_to_last_ratio": 0.7180297289018083, "imbalance_abs_log_ratio": 0.33124430563536084, "last_quartile_mean": 9.33926785364747e-05, "mean": 8.228966771639534e-05, "normalized": [ 0.8685826408392191, 0.8249006353131848, 0.7154905339803773, 0.6884730939760333, 0.9470487095509537, 0.8638069620260335, 0.8256155016985178, 0.7853664463613442, 0.9851686390305121, 0.9692511243547932, 0.9083804947955202, 0.8729855457896263, 1.0422548534804916, 1.0255492616133721, 0.9871528463964067, 0.9398223255303129, 1.0958375595009802, 1.0726325633807163, 1.0495259774381736, 1.0230838788771643, 1.1576441025653552, 1.1142532603054807, 1.1008567438213033, 1.056908774400763, 1.159063578749664, 1.1192434432436902, 1.1032241584740294, 1.1060435702772593, 1.1786235078649367, 1.1628365878275773, 1.141840825041133, 1.1085318534950752 ], "peak_layer": 29, "peak_normalized": 1.1786235078649367, "population_cv": 0.1341844256762087, "rest_mean": 8.049536754023629e-05, "spike_contrast": 1.1426606459269575, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.197888866765424e-05, "top_five_layers": [ 29, 30, 25, 21, 31 ], "top_five_spike_overlap": 2 }, "values": [ 7.14753768988885e-05, 6.788079917896539e-05, 5.887747829547152e-05, 5.66542221349664e-05, 7.793232362018898e-05, 7.108238787623122e-05, 6.793962529627606e-05, 6.462754390668124e-05, 8.106919995043427e-05, 7.975935295689851e-05, 7.475032907677814e-05, 7.183769048424438e-05, 8.576680556870997e-05, 8.4392107964959e-05, 8.123247971525416e-05, 7.733766688033938e-05, 9.017610864248127e-05, 8.826657722238451e-05, 8.636514394311234e-05, 8.418923243880272e-05, 9.526214853394777e-05, 9.169153054244816e-05, 9.0589135652408e-05, 8.697267185198143e-05, 9.537895675748587e-05, 9.210217103827745e-05, 9.078394941752777e-05, 9.101595787797123e-05, 9.698853682493791e-05, 9.56894364207983e-05, 9.396170207764953e-05, 9.122071787714958e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.05946834245696664, "first_to_last_ratio": 0.7180297222365345, "imbalance_abs_log_ratio": 0.33124431491808765, "last_quartile_mean": 0.08282156102359295, "mean": 0.07297530013602227, "normalized": [ 0.8685826122359729, 0.8249005917343738, 0.71549052911611, 0.6884731350635278, 0.9470486699630107, 0.8638070113577097, 0.8256154769796886, 0.7853664269052657, 0.9851686341386658, 0.9692511556795396, 0.9083805454214465, 0.8729855579949275, 1.0422548040385637, 1.0255492365750694, 0.9871528950273681, 0.9398223256645052, 1.0958375053304956, 1.07263252629438, 1.0495259690486791, 1.0230837911446407, 1.1576441435704576, 1.1142533045483842, 1.1008568124470575, 1.0569088284972943, 1.1590636022344982, 1.119243411839159, 1.1032241423789504, 1.1060435591240758, 1.1786235037173105, 1.1628366076958818, 1.141840809884862, 1.1085318743481278 ], "peak_layer": 29, "peak_normalized": 1.1786235037173105, "population_cv": 0.13418442668798974, "rest_mean": 0.07138409617322462, "spike_contrast": 1.1426607032635474, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.08156780153512955, "top_five_layers": [ 29, 30, 25, 21, 31 ], "top_five_spike_overlap": 2 }, "values": [ 0.06338507682085037, 0.0601973682641983, 0.05221313610672951, 0.050241533666849136, 0.0691111609339714, 0.06303657591342926, 0.060249537229537964, 0.05731235072016716, 0.07189297676086426, 0.07073139399290085, 0.06628934293985367, 0.06370638310909271, 0.07605885714292526, 0.0748397633433342, 0.07203777879476547, 0.06858381628990173, 0.07996907085180283, 0.07827568054199219, 0.07658947259187698, 0.07465984672307968, 0.0844794288277626, 0.08131296932697296, 0.0803353562951088, 0.07712823897600174, 0.08458301424980164, 0.08167712390422821, 0.08050811290740967, 0.08071386069059372, 0.0860104039311409, 0.08485835045576096, 0.08332617580890656, 0.08089544624090195 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 7.408341275549901e-06, "first_to_last_ratio": 0.49124437919492936, "imbalance_abs_log_ratio": 0.7108135577065074, "last_quartile_mean": 1.5080765479069669e-05, "mean": 1.1844782648040564e-05, "normalized": [ 0.5657990507718705, 0.5382233871697658, 0.49942145823009426, 0.48906984415931193, 0.8062921756393574, 0.7122953827323945, 0.7013508394120195, 0.6911626993599347, 0.8940675881372303, 0.868722127857376, 0.9735911331661544, 0.854855860776803, 0.989483587379368, 1.0059088486455374, 1.0303729006301217, 0.9941025538400318, 1.1620912657632405, 1.127012997009919, 1.113401270263767, 1.066097770938846, 1.1826034542812638, 1.1643547935214302, 1.2049778184223279, 1.1791491538213539, 1.2365172470752577, 1.219921447624247, 1.2337761202582145, 1.2288278254433027, 1.3039217543779833, 1.3631114810840155, 1.2856635723507386, 1.3138525898567213 ], "peak_layer": 30, "peak_normalized": 1.3631114810840155, "population_cv": 0.2549101540724733, "rest_mean": 1.1420299651271543e-05, "spike_contrast": 1.237882653019464, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.4136990830593277e-05, "top_five_layers": [ 30, 32, 29, 31, 25 ], "top_five_spike_overlap": 1 }, "values": [ 6.701766778860474e-06, 6.3751390371180605e-06, 5.915538622502936e-06, 5.7929260037781205e-06, 9.550355571263935e-06, 8.436983989668079e-06, 8.307348252856173e-06, 8.186671948351432e-06, 1.0590036254143342e-05, 1.0289824786013924e-05, 1.1531975360412616e-05, 1.0125581866304856e-05, 1.1720218026312068e-05, 1.1914771675947122e-05, 1.220454305439489e-05, 1.1774928680097219e-05, 1.3764718460151926e-05, 1.334922399109928e-05, 1.3187996046326589e-05, 1.2627696378331166e-05, 1.4007680874783546e-05, 1.379152945446549e-05, 1.4272700354922563e-05, 1.3966765436634887e-05, 1.4646278032159898e-05, 1.4449704394792207e-05, 1.4613809980801307e-05, 1.4555198504240252e-05, 1.5444669770658948e-05, 1.614575921848882e-05, 1.5228405572997872e-05, 1.5562298358418047e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 5.6029601637419546e-05, "first_to_last_ratio": 0.6070213107493427, "imbalance_abs_log_ratio": 0.49919138022039955, "last_quartile_mean": 9.230252817360451e-05, "mean": 7.840498903988191e-05, "normalized": [ 0.6598133090619259, 0.6419677920829977, 0.6005256373772865, 0.5913463573822231, 0.8562376513989928, 0.8062968559250673, 0.7923156566726156, 0.768439227921366, 0.9790122729268586, 0.9665730330981341, 0.9163710065350301, 0.8881686280668529, 1.0631190324760253, 1.0511030519715439, 1.0188080221730096, 0.9712812246823179, 1.1361272291219966, 1.1138561419734825, 1.0918134348286874, 1.0639217496772315, 1.203414696914147, 1.1592426077267617, 1.1442664103435203, 1.097952968622282, 1.2042281788778202, 1.1625445130141736, 1.1452986211445626, 1.1473932954411665, 1.2233942816523278, 1.205318760675343, 1.1829469858778916, 1.146901364356358 ], "peak_layer": 29, "peak_normalized": 1.2233942816523278, "population_cv": 0.1898597249088972, "rest_mean": 7.605543874820728e-05, "spike_contrast": 1.1977126437531995, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.109256061492487e-05, "top_five_layers": [ 29, 30, 25, 21, 31 ], "top_five_spike_overlap": 2 }, "values": [ 5.173265526536852e-05, 5.033347770222463e-05, 4.708420601673424e-05, 4.636450466932729e-05, 6.713330367347226e-05, 6.321769615169615e-05, 6.212150037754327e-05, 6.024946924299002e-05, 7.675944652874023e-05, 7.578414806630462e-05, 7.184805872384459e-05, 6.963685154914856e-05, 8.335383608937263e-05, 8.241172326961532e-05, 7.987963181221858e-05, 7.615329377586022e-05, 8.907804294722155e-05, 8.733187860343605e-05, 8.560362039133906e-05, 8.341677312273532e-05, 9.435371612198651e-05, 9.089040395338088e-05, 8.971619536168873e-05, 8.608499047113582e-05, 9.441749716643244e-05, 9.114928980125114e-05, 8.97971258382313e-05, 8.996135875349864e-05, 9.592021524440497e-05, 9.450300422031432e-05, 9.274894546251744e-05, 8.992278890218586e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 4.8511469685763586e-05, "first_to_last_ratio": 0.5271352332510719, "imbalance_abs_log_ratio": 0.6402981537748061, "last_quartile_mean": 9.202850924339145e-05, "mean": 7.532146378252946e-05, "normalized": [ 0.548104082267993, 0.5416830199005969, 0.528884702689083, 0.5247448675895723, 0.7777813723908718, 0.7487613756360456, 0.7496059866168319, 0.7329069161583202, 0.9480676391902623, 0.9399123881532756, 0.9000936195631334, 0.883441930780847, 1.0560589316963704, 1.0534552095740957, 1.032095781768693, 0.9835620691098637, 1.1601754560133901, 1.1467788545782247, 1.1233340534265408, 1.0967375255670042, 1.2396083970871608, 1.1951467072205837, 1.1828066060767042, 1.1317734013234224, 1.244698956870482, 1.2025862582796845, 1.1881970083126314, 1.1882256015362047, 1.2689460104605979, 1.2589162611011824, 1.2307315535498626, 1.1921774555104672 ], "peak_layer": 29, "peak_normalized": 1.2689460104605979, "population_cv": 0.23486334953575583, "rest_mean": 7.254842300123225e-05, "spike_contrast": 1.244629176846487, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.029588400153444e-05, "top_five_layers": [ 29, 30, 25, 21, 31 ], "top_five_spike_overlap": 2 }, "values": [ 4.1284001781605184e-05, 4.080035796505399e-05, 3.9836369978729635e-05, 3.952455153921619e-05, 5.858363147126511e-05, 5.639780283672735e-05, 5.6461420172126964e-05, 5.520362174138427e-05, 7.140984234865755e-05, 7.079557690303773e-05, 6.77963689668104e-05, 6.654213939327747e-05, 7.954390457598493e-05, 7.934778841445222e-05, 7.773896504659206e-05, 7.408333476632833e-05, 8.738611359149218e-05, 8.637706196168438e-05, 8.461116522084922e-05, 8.26078758109361e-05, 9.336911898571998e-05, 9.002019942272455e-05, 8.909072494134307e-05, 8.524682925781235e-05, 9.375254740007222e-05, 9.058055729838088e-05, 8.949673792812973e-05, 8.949889161158353e-05, 9.557887096889317e-05, 9.482341556577012e-05, 9.270050213672221e-05, 8.979655103757977e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 9.61669284151867e-05, "first_to_last_ratio": 0.8297833624837898, "imbalance_abs_log_ratio": 0.18659062131596024, "last_quartile_mean": 0.00011589401856326731, "mean": 0.00010716335873439675, "normalized": [ 0.9380742209455786, 0.8898820858744699, 0.7837417114644826, 0.7583102664636657, 1.0822070619439073, 0.9520749081713696, 0.9107982467806989, 0.8640017807748708, 1.0669538303069324, 1.0266950649381863, 0.9594029855154438, 0.8901506143407661, 1.0795289063552718, 1.0379165674077444, 0.9706971366471856, 0.914979822300863, 1.052939971267624, 1.0225707935964177, 0.9851238030379684, 0.9636652893153618, 1.0990044510799908, 1.0523904893769196, 1.0422234788441622, 1.0049019599938585, 1.0962247910521359, 1.0599378706264646, 1.05039930496957, 1.0583562345651827, 1.1199788667078585, 1.1100529546792268, 1.0850973923636869, 1.0717171382921347 ], "peak_layer": 29, "peak_normalized": 1.1199788667078585, "population_cv": 0.09156717959150794, "rest_mean": 0.00010599351123716958, "spike_contrast": 1.0706366257223145, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00011348053521942347, "top_five_layers": [ 29, 30, 21, 25, 31 ], "top_five_spike_overlap": 2 }, "values": [ 0.00010052718425868079, 9.536275319987908e-05, 8.398839418077841e-05, 8.12630751170218e-05, 0.00011597294360399246, 0.00010202754492638633, 9.760419925441965e-05, 9.25893327803351e-05, 0.00011433835607022047, 0.00011002409155480564, 0.00010281284630764276, 9.539152961224318e-05, 0.000115685943455901, 0.0001112266254494898, 0.0001040231654769741, 9.805231093196198e-05, 0.00011283658386673778, 0.00010958212078548968, 0.00010556917550275102, 0.00010326960909878835, 0.00011777300824178383, 0.0001127776995417662, 0.00011168816854478791, 0.00010768866923172027, 0.00011747513053705916, 0.00011358650226611644, 0.00011256431753281504, 0.00011341700883349404, 0.00012002069706795737, 0.00011895700299646705, 0.00011628268111962825, 0.00011484880815260112 ] } } } }, "selector": { "expected_uniform_count": 4, "identities_unique": true, "identity_order_sha256": "97fd9721a263024f6a746638f32e0ad9a988cd4c111c00e9e178c1022e5c9d5e", "passed": true, "uniform_count": 4, "uniform_indices": [ 40, 42, 44, 46 ], "visit_count": 65, "visits": [ { "branch": "attention", "coefficient": "detached_learned", "group": 1, "index": 0, "kind": "depth", "layer": 1, "offset": 1, "sources": 1, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 1, "index": 1, "kind": "depth", "layer": 1, "offset": 1, "sources": 2, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 1, "index": 2, "kind": "depth", "layer": 2, "offset": 2, "sources": 2, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 1, "index": 3, "kind": "depth", "layer": 2, "offset": 2, "sources": 2, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 1, "index": 4, "kind": "depth", "layer": 3, "offset": 3, "sources": 2, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 1, "index": 5, "kind": "depth", "layer": 3, "offset": 3, "sources": 2, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 1, "index": 6, "kind": "depth", "layer": 4, "offset": 4, "sources": 2, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 1, "index": 7, "kind": "depth", "layer": 4, "offset": 4, "sources": 2, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 2, "index": 8, "kind": "depth", "layer": 5, "offset": 1, "sources": 2, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 2, "index": 9, "kind": "depth", "layer": 5, "offset": 1, "sources": 3, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 2, "index": 10, "kind": "depth", "layer": 6, "offset": 2, "sources": 3, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 2, "index": 11, "kind": "depth", "layer": 6, "offset": 2, "sources": 3, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 2, "index": 12, "kind": "depth", "layer": 7, "offset": 3, "sources": 3, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 2, "index": 13, "kind": "depth", "layer": 7, "offset": 3, "sources": 3, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 2, "index": 14, "kind": "depth", "layer": 8, "offset": 4, "sources": 3, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 2, "index": 15, "kind": "depth", "layer": 8, "offset": 4, "sources": 3, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 3, "index": 16, "kind": "depth", "layer": 9, "offset": 1, "sources": 3, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 3, "index": 17, "kind": "depth", "layer": 9, "offset": 1, "sources": 4, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 3, "index": 18, "kind": "depth", "layer": 10, "offset": 2, "sources": 4, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 3, "index": 19, "kind": "depth", "layer": 10, "offset": 2, "sources": 4, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 3, "index": 20, "kind": "depth", "layer": 11, "offset": 3, "sources": 4, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 3, "index": 21, "kind": "depth", "layer": 11, "offset": 3, "sources": 4, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 3, "index": 22, "kind": "depth", "layer": 12, "offset": 4, "sources": 4, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 3, "index": 23, "kind": "depth", "layer": 12, "offset": 4, "sources": 4, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 4, "index": 24, "kind": "depth", "layer": 13, "offset": 1, "sources": 4, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 4, "index": 25, "kind": "depth", "layer": 13, "offset": 1, "sources": 5, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 4, "index": 26, "kind": "depth", "layer": 14, "offset": 2, "sources": 5, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 4, "index": 27, "kind": "depth", "layer": 14, "offset": 2, "sources": 5, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 4, "index": 28, "kind": "depth", "layer": 15, "offset": 3, "sources": 5, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 4, "index": 29, "kind": "depth", "layer": 15, "offset": 3, "sources": 5, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 4, "index": 30, "kind": "depth", "layer": 16, "offset": 4, "sources": 5, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 4, "index": 31, "kind": "depth", "layer": 16, "offset": 4, "sources": 5, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 5, "index": 32, "kind": "depth", "layer": 17, "offset": 1, "sources": 5, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 5, "index": 33, "kind": "depth", "layer": 17, "offset": 1, "sources": 6, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 5, "index": 34, "kind": "depth", "layer": 18, "offset": 2, "sources": 6, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 5, "index": 35, "kind": "depth", "layer": 18, "offset": 2, "sources": 6, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 5, "index": 36, "kind": "depth", "layer": 19, "offset": 3, "sources": 6, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 5, "index": 37, "kind": "depth", "layer": 19, "offset": 3, "sources": 6, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 5, "index": 38, "kind": "depth", "layer": 20, "offset": 4, "sources": 6, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 5, "index": 39, "kind": "depth", "layer": 20, "offset": 4, "sources": 6, "use_uniform": false }, { "branch": "attention", "coefficient": "uniform", "group": 6, "index": 40, "kind": "depth", "layer": 21, "offset": 1, "sources": 6, "use_uniform": true }, { "branch": "mlp", "coefficient": "detached_learned", "group": 6, "index": 41, "kind": "depth", "layer": 21, "offset": 1, "sources": 7, "use_uniform": false }, { "branch": "attention", "coefficient": "uniform", "group": 6, "index": 42, "kind": "depth", "layer": 22, "offset": 2, "sources": 7, "use_uniform": true }, { "branch": "mlp", "coefficient": "detached_learned", "group": 6, "index": 43, "kind": "depth", "layer": 22, "offset": 2, "sources": 7, "use_uniform": false }, { "branch": "attention", "coefficient": "uniform", "group": 6, "index": 44, "kind": "depth", "layer": 23, "offset": 3, "sources": 7, "use_uniform": true }, { "branch": "mlp", "coefficient": "detached_learned", "group": 6, "index": 45, "kind": "depth", "layer": 23, "offset": 3, "sources": 7, "use_uniform": false }, { "branch": "attention", "coefficient": "uniform", "group": 6, "index": 46, "kind": "depth", "layer": 24, "offset": 4, "sources": 7, "use_uniform": true }, { "branch": "mlp", "coefficient": "detached_learned", "group": 6, "index": 47, "kind": "depth", "layer": 24, "offset": 4, "sources": 7, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 7, "index": 48, "kind": "depth", "layer": 25, "offset": 1, "sources": 7, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 7, "index": 49, "kind": "depth", "layer": 25, "offset": 1, "sources": 8, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 7, "index": 50, "kind": "depth", "layer": 26, "offset": 2, "sources": 8, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 7, "index": 51, "kind": "depth", "layer": 26, "offset": 2, "sources": 8, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 7, "index": 52, "kind": "depth", "layer": 27, "offset": 3, "sources": 8, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 7, "index": 53, "kind": "depth", "layer": 27, "offset": 3, "sources": 8, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 7, "index": 54, "kind": "depth", "layer": 28, "offset": 4, "sources": 8, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 7, "index": 55, "kind": "depth", "layer": 28, "offset": 4, "sources": 8, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 8, "index": 56, "kind": "depth", "layer": 29, "offset": 1, "sources": 8, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 8, "index": 57, "kind": "depth", "layer": 29, "offset": 1, "sources": 9, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 8, "index": 58, "kind": "depth", "layer": 30, "offset": 2, "sources": 9, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 8, "index": 59, "kind": "depth", "layer": 30, "offset": 2, "sources": 9, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 8, "index": 60, "kind": "depth", "layer": 31, "offset": 3, "sources": 9, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 8, "index": 61, "kind": "depth", "layer": 31, "offset": 3, "sources": 9, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 8, "index": 62, "kind": "depth", "layer": 32, "offset": 4, "sources": 9, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 8, "index": 63, "kind": "depth", "layer": 32, "offset": 4, "sources": 9, "use_uniform": false }, { "branch": "output", "coefficient": "detached_learned", "group": null, "index": 64, "kind": "output", "layer": null, "offset": null, "sources": 9, "use_uniform": false } ], "visits_sha256": "405951ed108e29cdea8fcb883903e04869c6b74f99f62584e8719da66082d8cf" } }, "uniform_group_6_mlp_only": { "bits_per_byte": 7.959341947457508, "forward": { "activation_sha256": { "attention_branch_output": "676759a2d7c0615ad7e1f246e90927fb829d00c8b6c3745032ae6b27ab7d902a", "mlp_branch_output": "afe29f3ef4d2992353f0bb59fe7c5c2c5050fca671a53184825a8d63cd83a2bc", "post_attention_state": "31853ffb07b24cc59e774bb20656840b4358b3080b42516d380f2dc0f46404c8", "post_mlp_state": "7fd9c911e93175896ad618842c120e77b3f017c17b7eec9c2bfb01fc6dbc8ac2", "pre_attention_input": "de71d4cf2d509a1063f6eac7f50c4885e9a058dee7e4b993ad40f5527f528d00", "pre_mlp_input": "2999a257265024350fd2665e2a8d452a03cd662193b4245d15290408ad29922d" }, "logits_sha256": "873a309a197bfd08fa523edcb440d2697903a34abc02dad1c7f5991509c3c151", "loss_nats": 5.516995429992676, "mixer_summary_sha256": "8450b24bdc777885a3357dc7b8080e7410acbe96943686433da76f379295133a" }, "loss_nats": 5.516995429992676, "loss_scale": 1.0, "mixers": [ { "branch": "attention", "entropy_mean": -0.0, "group": 1, "latest_source_mass": 1.0, "layer": 1, "max_source_mass": 1.0, "mixer_index": 0, "normalized_entropy": 1.0, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 1.0, "median": 1.0, "p05": 1.0, "p95": 1.0 } ], "sources": 1 }, { "branch": "mlp", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 1, "max_source_mass": 0.5, "mixer_index": 1, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 2, "max_source_mass": 0.5, "mixer_index": 2, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 2, "max_source_mass": 0.5, "mixer_index": 3, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 3, "max_source_mass": 0.5, "mixer_index": 4, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 3, "max_source_mass": 0.5, "mixer_index": 5, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 4, "max_source_mass": 0.5, "mixer_index": 6, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 4, "max_source_mass": 0.5, "mixer_index": 7, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6931471824645996, "group": 2, "latest_source_mass": 0.5, "layer": 5, "max_source_mass": 0.5, "mixer_index": 8, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "completed_group_1", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 5, "max_source_mass": 0.3333333432674408, "mixer_index": 9, "normalized_entropy": 1.0000000180537645, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 6, "max_source_mass": 0.3333333432674408, "mixer_index": 10, "normalized_entropy": 1.0000000180537645, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 6, "max_source_mass": 0.3333333432674408, "mixer_index": 11, "normalized_entropy": 1.0000000180537645, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 7, "max_source_mass": 0.3333333432674408, "mixer_index": 12, "normalized_entropy": 1.0000000180537645, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 7, "max_source_mass": 0.3333333432674408, "mixer_index": 13, "normalized_entropy": 1.0000000180537645, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 8, "max_source_mass": 0.3333333432674408, "mixer_index": 14, "normalized_entropy": 1.0000000180537645, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 8, "max_source_mass": 0.3333333432674408, "mixer_index": 15, "normalized_entropy": 1.0000000180537645, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0986123085021973, "group": 3, "latest_source_mass": 0.3333333432674408, "layer": 9, "max_source_mass": 0.3333333432674408, "mixer_index": 16, "normalized_entropy": 1.0000000180537645, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_2", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 9, "max_source_mass": 0.25, "mixer_index": 17, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 10, "max_source_mass": 0.25, "mixer_index": 18, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 10, "max_source_mass": 0.25, "mixer_index": 19, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 11, "max_source_mass": 0.25, "mixer_index": 20, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 11, "max_source_mass": 0.25, "mixer_index": 21, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 12, "max_source_mass": 0.25, "mixer_index": 22, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 12, "max_source_mass": 0.25, "mixer_index": 23, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3862943649291992, "group": 4, "latest_source_mass": 0.25, "layer": 13, "max_source_mass": 0.25, "mixer_index": 24, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_3", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 13, "max_source_mass": 0.20000000298023224, "mixer_index": 25, "normalized_entropy": 1.0000000186840277, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 14, "max_source_mass": 0.20000000298023224, "mixer_index": 26, "normalized_entropy": 1.0000000186840277, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 14, "max_source_mass": 0.20000000298023224, "mixer_index": 27, "normalized_entropy": 1.0000000186840277, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 15, "max_source_mass": 0.20000000298023224, "mixer_index": 28, "normalized_entropy": 1.0000000186840277, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 15, "max_source_mass": 0.20000000298023224, "mixer_index": 29, "normalized_entropy": 1.0000000186840277, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 16, "max_source_mass": 0.20000000298023224, "mixer_index": 30, "normalized_entropy": 1.0000000186840277, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 16, "max_source_mass": 0.20000000298023224, "mixer_index": 31, "normalized_entropy": 1.0000000186840277, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.6094379425048828, "group": 5, "latest_source_mass": 0.20000000298023224, "layer": 17, "max_source_mass": 0.20000000298023224, "mixer_index": 32, "normalized_entropy": 1.0000000186840277, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_4", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 17, "max_source_mass": 0.1666666716337204, "mixer_index": 33, "normalized_entropy": 1.0000001451965654, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 18, "max_source_mass": 0.1666666716337204, "mixer_index": 34, "normalized_entropy": 1.0000001451965654, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 18, "max_source_mass": 0.1666666716337204, "mixer_index": 35, "normalized_entropy": 1.0000001451965654, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 19, "max_source_mass": 0.1666666716337204, "mixer_index": 36, "normalized_entropy": 1.0000001451965654, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 19, "max_source_mass": 0.1666666716337204, "mixer_index": 37, "normalized_entropy": 1.0000001451965654, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 20, "max_source_mass": 0.1666666716337204, "mixer_index": 38, "normalized_entropy": 1.0000001451965654, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 20, "max_source_mass": 0.1666666716337204, "mixer_index": 39, "normalized_entropy": 1.0000001451965654, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.791759729385376, "group": 6, "latest_source_mass": 0.1666666716337204, "layer": 21, "max_source_mass": 0.1666666716337204, "mixer_index": 40, "normalized_entropy": 1.0000001451965654, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_5", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 21, "max_source_mass": 0.1428571492433548, "mixer_index": 41, "normalized_entropy": 1.00000003408302, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 22, "max_source_mass": 0.1428571492433548, "mixer_index": 42, "normalized_entropy": 1.00000003408302, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 22, "max_source_mass": 0.1428571492433548, "mixer_index": 43, "normalized_entropy": 1.00000003408302, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 23, "max_source_mass": 0.1428571492433548, "mixer_index": 44, "normalized_entropy": 1.00000003408302, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 23, "max_source_mass": 0.1428571492433548, "mixer_index": 45, "normalized_entropy": 1.00000003408302, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 24, "max_source_mass": 0.1428571492433548, "mixer_index": 46, "normalized_entropy": 1.00000003408302, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 24, "max_source_mass": 0.1428571492433548, "mixer_index": 47, "normalized_entropy": 1.00000003408302, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9459102153778076, "group": 7, "latest_source_mass": 0.1428571492433548, "layer": 25, "max_source_mass": 0.1428571492433548, "mixer_index": 48, "normalized_entropy": 1.00000003408302, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_6", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 25, "max_source_mass": 0.125, "mixer_index": 49, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 26, "max_source_mass": 0.125, "mixer_index": 50, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 26, "max_source_mass": 0.125, "mixer_index": 51, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 27, "max_source_mass": 0.125, "mixer_index": 52, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 27, "max_source_mass": 0.125, "mixer_index": 53, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 28, "max_source_mass": 0.125, "mixer_index": 54, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 28, "max_source_mass": 0.125, "mixer_index": 55, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.079441547393799, "group": 8, "latest_source_mass": 0.125, "layer": 29, "max_source_mass": 0.125, "mixer_index": 56, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_7", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 29, "max_source_mass": 0.1111111119389534, "mixer_index": 57, "normalized_entropy": 1.0000000180537645, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 30, "max_source_mass": 0.1111111119389534, "mixer_index": 58, "normalized_entropy": 1.0000000180537645, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 30, "max_source_mass": 0.1111111119389534, "mixer_index": 59, "normalized_entropy": 1.0000000180537645, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 31, "max_source_mass": 0.1111111119389534, "mixer_index": 60, "normalized_entropy": 1.0000000180537645, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 31, "max_source_mass": 0.1111111119389534, "mixer_index": 61, "normalized_entropy": 1.0000000180537645, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 32, "max_source_mass": 0.1111111119389534, "mixer_index": 62, "normalized_entropy": 1.0000000180537645, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 32, "max_source_mass": 0.1111111119389534, "mixer_index": 63, "normalized_entropy": 1.0000000180537645, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "output", "entropy_mean": 2.1972246170043945, "group": null, "latest_source_mass": 0.1111111119389534, "layer": null, "max_source_mass": 0.1111111119389534, "mixer_index": null, "normalized_entropy": 1.0000000180537645, "offset": null, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_8", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 } ], "mode": "uniform_group_6_mlp_only", "optimizer_state_after": "b82a349700eda58e2330a22ca2ee1f7fb1406efd47fd9aa306122b368a114192", "optimizer_state_before": "b82a349700eda58e2330a22ca2ee1f7fb1406efd47fd9aa306122b368a114192", "optimizer_state_unchanged": true, "positions": { "attention_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "676759a2d7c0615ad7e1f246e90927fb829d00c8b6c3745032ae6b27ab7d902a", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.0022265110164880753, 0.0021613112185150385, 0.0025601957459002733, 0.0025323182344436646, 0.0027253651060163975, 0.002904455177485943, 0.0029208383057266474, 0.00299624796025455, 0.0030937574338167906, 0.0033804215490818024, 0.0034370317589491606, 0.0038969130255281925, 0.004255065228790045, 0.004119740333408117, 0.003970506135374308, 0.004385598469525576, 0.0041999719105660915, 0.005232699681073427, 0.004664790350943804, 0.004898969549685717, 0.005131533369421959, 0.005069608334451914, 0.0051318020559847355, 0.0058372304774820805, 0.0059690922498703, 0.00554921617731452, 0.0058550527319312096, 0.006255472078919411, 0.006502739619463682, 0.006122888531535864, 0.007071305997669697, 0.006241494789719582 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.00015962477027642308, "first_to_last_ratio": 3.7699040072151497, "imbalance_abs_log_ratio": 1.3270495388586014, "last_quartile_mean": 4.234186599205714e-05, "mean": 8.617327489446325e-05, "normalized": [ 2.5428430303860874, 2.3222064178412967, 2.1011513502014654, 1.8911470095267005, 1.7004447966680725, 1.5353700031893112, 1.4229335846259188, 1.3028620492588128, 1.2049362956920768, 1.1705197150696323, 1.0773612761873523, 1.0133200843661747, 0.9906112963967714, 0.9272530070793641, 0.8550207003106389, 0.8185600560490035, 0.7703816559177782, 0.7173918665339755, 0.6760157258502337, 0.6514535263006792, 0.6184138115448249, 0.6053606047045522, 0.5838821473278404, 0.5697017460729616, 0.5457761369570343, 0.5276086337508958, 0.5104655265174319, 0.4951752820428687, 0.47527333481689216, 0.465789239220051, 0.45896414036514044, 0.45180594922816086 ], "peak_layer": 1, "peak_normalized": 2.5428430303860874, "population_cv": 0.5705341151299013, "rest_mean": 9.280180475562466e-05, "spike_contrast": 0.5428688997682254, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.0379213644191625e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00021912511147093028, 0.00020011213200632483, 0.00018106309289578348, 0.00016296633111778647, 0.00014653289690613747, 0.00013230786134954542, 0.0001226188469445333, 0.00011227188952034339, 0.0001038333066389896, 0.00010086751717608422, 9.283974941354245e-05, 8.732111018616706e-05, 8.536421955795959e-05, 7.990442827576771e-05, 7.367993384832516e-05, 7.053800072753802e-05, 6.63863102090545e-05, 6.182000652188435e-05, 5.825448897667229e-05, 5.6137883802875876e-05, 5.329074338078499e-05, 5.216590579948388e-05, 5.031503678765148e-05, 4.909306517220102e-05, 4.703131708083674e-05, 4.5465763832908124e-05, 4.3988486140733585e-05, 4.2670875700423494e-05, 4.095585973118432e-05, 4.013858415419236e-05, 3.955044303438626e-05, 3.893359826179221e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.00021442661636683624, "first_to_last_ratio": 1.843144421770586, "imbalance_abs_log_ratio": 0.6114730379455638, "last_quartile_mean": 0.0001163373926829081, "mean": 0.00015221274247778638, "normalized": [ 1.7741446897870732, 1.648461275666186, 1.5301464224684427, 1.4217121808823485, 1.3265065496562456, 1.2454420544341354, 1.1897998831173078, 1.1336248235930768, 1.0869089599614004, 1.0643003125599004, 1.0227030055526172, 0.9941765648180964, 0.9788958472869692, 0.9511102301475627, 0.9228060651723192, 0.9052418824754537, 0.8846484396889256, 0.865292475366091, 0.8497311645646503, 0.8388691935254817, 0.8256303537004397, 0.8176325372729673, 0.8076852907347736, 0.8000670639426607, 0.7892588692968658, 0.7814415503461742, 0.7731202151361662, 0.7664994567889658, 0.758490072461763, 0.7529625284402992, 0.7485174912671357, 0.7441725498875049 ], "peak_layer": 1, "peak_normalized": 1.7741446897870732, "population_cv": 0.27364872432853743, "rest_mean": 0.00015762320577373935, "spike_contrast": 0.7803181014868814, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012299624067964033, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00027004742878489196, 0.0002509168116375804, 0.0002329077833564952, 0.00021640271006617695, 0.00020191119983792305, 0.00018957215070258826, 0.0001811027032090351, 0.00017255214333999902, 0.00016544139361940324, 0.00016200006939470768, 0.00015566842921543866, 0.0001513263414381072, 0.00014900042151566595, 0.0001447710965294391, 0.00014046284195501357, 0.0001377893495373428, 0.00013465476513374597, 0.00013170854072086513, 0.00012933991092722863, 0.0001276865805266425, 0.00012567146040964872, 0.00012445409083738923, 0.00012293999316170812, 0.00012178040196886286, 0.00012013525702059269, 0.00011894536146428436, 0.00011767874821089208, 0.00011667098442558199, 0.00011545185407157987, 0.00011461049143690616, 0.00011393390013836324, 0.00011327254469506443 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.19015566632151604, "first_to_last_ratio": 1.8431444167877922, "imbalance_abs_log_ratio": 0.6114730352421437, "last_quartile_mean": 0.10316916275769472, "mean": 0.13498378219082952, "normalized": [ 1.7741447141792424, 1.6484611693507762, 1.5301464164964773, 1.421712099711509, 1.3265065258282553, 1.245442086656075, 1.1897999853143832, 1.1336247998774232, 1.0869090145314644, 1.0643003539468234, 1.0227030148964142, 0.9941765578539046, 0.978895844117059, 0.9511102293531465, 0.9228060470726628, 0.9052419251832852, 0.8846484748200404, 0.8652925213936268, 0.8497311905993837, 0.8388692019864461, 0.8256303029992552, 0.8176324960789627, 0.8076852727084025, 0.800067049482784, 0.789258877083647, 0.7814415614650243, 0.7731201949085273, 0.7664994759042474, 0.758490077799444, 0.7529625178867759, 0.7485174642151933, 0.7441725362993381 ], "peak_layer": 1, "peak_normalized": 1.7741447141792424, "population_cv": 0.2736487184871982, "rest_mean": 0.13978183573042904, "spike_contrast": 0.7803180757143806, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.10907429307699204, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.23948076367378235, 0.2225155234336853, 0.20654495060443878, 0.1919080764055252, 0.17905686795711517, 0.16811448335647583, 0.16060370206832886, 0.1530209630727768, 0.14671508967876434, 0.14366328716278076, 0.13804832100868225, 0.13419771194458008, 0.1321350634098053, 0.12838445603847504, 0.12456385046243668, 0.12219297885894775, 0.11941319704055786, 0.116800457239151, 0.11469992995262146, 0.11323373764753342, 0.11144670099020004, 0.11036712676286697, 0.10902441293001175, 0.10799607634544373, 0.10653714835643768, 0.10548193752765656, 0.10435868799686432, 0.1034649983048439, 0.10238385945558548, 0.10163772851228714, 0.10103771835565567, 0.10045122355222702 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 6.883365585963475e-05, "first_to_last_ratio": 2.2497546817252747, "imbalance_abs_log_ratio": 0.8108211799277664, "last_quartile_mean": 3.059607183786284e-05, "mean": 4.691841348858361e-05, "normalized": [ 1.7359303646705315, 1.6475859007944922, 1.5536416135247637, 1.4921074478374423, 1.436565604934822, 1.343834173349448, 1.2787439414688093, 1.2483315192759759, 1.191227587726315, 1.1603086071531024, 1.0995306735006558, 1.059977946939621, 1.0583984121521266, 1.0176934649371145, 0.9754909410255406, 0.9631621800279858, 0.9242301483750994, 0.8752327792179452, 0.8454160845678106, 0.8232909667825862, 0.7863453139125789, 0.77746685672799, 0.7497386589361914, 0.7388508716017261, 0.7093180450413769, 0.6908354487677191, 0.6685471901846501, 0.6592354484095886, 0.631440334976849, 0.6235901145414114, 0.6199506557967245, 0.6139807028410055 ], "peak_layer": 1, "peak_normalized": 1.7359303646705315, "population_cv": 0.3220412433027962, "rest_mean": 4.907019663530971e-05, "spike_contrast": 0.7193528234378936, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.529878449626267e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 8.144709863699973e-05, 7.730211655143648e-05, 7.289439963642508e-05, 7.000731420703232e-05, 6.740137905580923e-05, 6.305056740529835e-05, 5.999663699185476e-05, 5.856973439222202e-05, 5.5890508519951254e-05, 5.4439839004771784e-05, 5.1588234782684594e-05, 4.9732483603293076e-05, 4.965837433701381e-05, 4.774856279254891e-05, 4.576848732540384e-05, 4.519004141911864e-05, 4.3363412260077894e-05, 4.106453343410976e-05, 3.966558142565191e-05, 3.862750600092113e-05, 3.6894074582960457e-05, 3.647751145763323e-05, 3.517654840834439e-05, 3.4665710700210184e-05, 3.328007733216509e-05, 3.241290323785506e-05, 3.136717350571416e-05, 3.0930281354812905e-05, 2.9626178729813546e-05, 2.925785884144716e-05, 2.9087101211189292e-05, 2.8807000489905477e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 0.0001734125216898974, "first_to_last_ratio": 1.495517223041696, "imbalance_abs_log_ratio": 0.4024721155990803, "last_quartile_mean": 0.00011595488103921525, "mean": 0.00013990532806928968, "normalized": [ 1.3768889945483427, 1.3340940553978193, 1.2910049686488183, 1.2527203273458163, 1.2147536525239127, 1.1769854337196524, 1.14744939519443, 1.1220955988146075, 1.0944022472959734, 1.0773022658134643, 1.0508559212107247, 1.03206209473486, 1.0208171898483585, 1.000561882500425, 0.9805161605170362, 0.9663129296065233, 0.9487424991891081, 0.931931984738623, 0.9181974345490227, 0.9075104531838204, 0.8943640941785878, 0.8862664021756573, 0.8757830774574916, 0.86790401991425, 0.8562894022841879, 0.8478155487572495, 0.8387227683262297, 0.8314011142413483, 0.8224071457687988, 0.8161875050523302, 0.8112199678685613, 0.8064334645939688 ], "peak_layer": 1, "peak_normalized": 1.3768889945483427, "population_cv": 0.16325957486062354, "rest_mean": 0.0001431148236776978, "spike_contrast": 0.8564734849544942, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001225740517838858, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0001926341064972803, 0.00018664686649572104, 0.00018061847367789596, 0.00017526224837638438, 0.0001699505082797259, 0.00016466653323732316, 0.00016053428407758474, 0.00015698715287726372, 0.00015311270544771105, 0.00015072032692842185, 0.00014702034241054207, 0.00014439098595175892, 0.00014281776384450495, 0.00013998393842484802, 0.00013717943511437625, 0.00013519232743419707, 0.00013273413060232997, 0.00013038225006312132, 0.00012846071331296116, 0.00012696554767899215, 0.0001251263020094484, 0.00012399339175317436, 0.0001225267187692225, 0.00012142439663875848, 0.00011979944974882528, 0.00011861391249112785, 0.00011734178406186402, 0.00011631744564510882, 0.00011505914153531194, 0.00011418898066040128, 0.00011349399574100971, 0.00011282433843007311 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 0.00014675117381557357, "first_to_last_ratio": 1.2676837465168667, "imbalance_abs_log_ratio": 0.23719141364257146, "last_quartile_mean": 0.00011576323686313117, "mean": 0.00013042020850662084, "normalized": [ 1.169988069421413, 1.156175152925158, 1.1417665260225645, 1.1306726331060184, 1.1183621029407895, 1.1062694829504214, 1.0940115056230622, 1.0844989956600726, 1.0711591662817939, 1.0579765491085218, 1.0443900226505625, 1.035471661013805, 1.0260498632741444, 1.013992390086173, 1.0062479316494695, 0.9977001177294099, 0.9863163473759529, 0.9769959732660659, 0.9687585669786032, 0.9602628595878113, 0.9503845997255937, 0.9426104617668664, 0.9330243108845692, 0.9259760938667744, 0.914797625470175, 0.906289421438531, 0.8956357003805738, 0.888470885240542, 0.8806216558595509, 0.8756991509238615, 0.8712858841028995, 0.8681382926882502 ], "peak_layer": 1, "peak_normalized": 1.169988069421413, "population_cv": 0.09123006364857297, "rest_mean": 0.00013202972385678786, "spike_contrast": 0.9219804606102014, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012172882561571896, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00015259008796419948, 0.00015078860451467335, 0.00014890942838974297, 0.0001474625605624169, 0.00014585701865144074, 0.00014427989663090557, 0.00014268120867200196, 0.00014144058513920754, 0.00013970080181024969, 0.0001379815221298486, 0.00013620956451632082, 0.00013504642993211746, 0.0001338176371064037, 0.00013224509893916547, 0.00013123506505507976, 0.0001301202573813498, 0.00012863558367826045, 0.00012742001854348928, 0.00012634569429792464, 0.00012523768236860633, 0.00012394935765769333, 0.00012293545296415687, 0.00012168522516731173, 0.00012076599523425102, 0.00011930809705518186, 0.00011819845531135798, 0.00011680899478960782, 0.00011587455810513347, 0.00011485085997264832, 0.00011420886585256085, 0.00011363328667357564, 0.00011322277714498341 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00030002388848515693, "first_to_last_ratio": 2.2836575149168885, "imbalance_abs_log_ratio": 0.8257783308640597, "last_quartile_mean": 0.0001313786706305109, "mean": 0.0001947244400071213, "normalized": [ 1.8998833386658438, 1.7835645331795598, 1.6713073472446418, 1.5703972875779748, 1.4780250735507643, 1.3761669788219328, 1.3066447563403325, 1.2401013786965793, 1.1762106459027857, 1.150547637187856, 1.0768735950666006, 1.0264900078881174, 1.0140784159549234, 0.9666871251601796, 0.9096841069789114, 0.8754807117769676, 0.8421876125778792, 0.8073809902734768, 0.77911667677541, 0.7607651063686129, 0.7403822030704592, 0.7308855611321243, 0.7152791523187013, 0.7043381880055872, 0.6944157299089508, 0.686864629891504, 0.680599049276497, 0.6764772714226681, 0.6702412094778135, 0.6672258960044359, 0.6635863560708457, 0.6581114274310631 ], "peak_layer": 1, "peak_normalized": 1.8998833386658438, "population_cv": 0.3592853832280775, "rest_mean": 0.00020492727344390005, "spike_contrast": 0.6813594750078023, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013962913944851608, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0003699537192005664, 0.0003473036049399525, 0.0003254443872720003, 0.00030579473241232336, 0.00028780760476365685, 0.00026797334430739284, 0.0002544356684666127, 0.0002414780465187505, 0.0002290369593538344, 0.00022403974435292184, 0.00020969360775779933, 0.00019988269195891917, 0.0001974658516701311, 0.00018823760910890996, 0.00017713772831484675, 0.00017047749133780599, 0.00016399451124016196, 0.0001572168112033978, 0.00015171305858530104, 0.00014813955931458622, 0.00014417050988413393, 0.00014232128160074353, 0.00013928233238402754, 0.0001371518592350185, 0.00013521971413865685, 0.00013374933041632175, 0.00013252926873974502, 0.00013172665785532445, 0.00013051234418526292, 0.00012992518895771354, 0.00012921648158226162, 0.000128150379168801 ] } } }, "mlp_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "afe29f3ef4d2992353f0bb59fe7c5c2c5050fca671a53184825a8d63cd83a2bc", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.0027073738165199757, 0.002722575329244137, 0.0026850115973502398, 0.0026861224323511124, 0.00271838391199708, 0.0026645169127732515, 0.0026859452482312918, 0.0027359756641089916, 0.0026767742820084095, 0.0027451682835817337, 0.0026999011170119047, 0.002663193503394723, 0.002626648638397455, 0.0026531771291047335, 0.0026517712976783514, 0.002676903735846281, 0.002670086920261383, 0.0026355870068073273, 0.0026610493659973145, 0.0026388936676084995, 0.0026469926815479994, 0.00264862016774714, 0.002641494618728757, 0.002674198243767023, 0.002557571977376938, 0.00252733426168561, 0.0025468147359788418, 0.0026856842450797558, 0.002551089273765683, 0.0025778315030038357, 0.0026001501828432083, 0.0025662274565547705 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.00015858857113926206, "first_to_last_ratio": 3.757867690304466, "imbalance_abs_log_ratio": 1.3238516928952826, "last_quartile_mean": 4.220174423608114e-05, "mean": 8.573221236929385e-05, "normalized": [ 2.5461641896458365, 2.3032502265274117, 2.0924715218207024, 1.8886722264368014, 1.7031663195050915, 1.537434754996766, 1.4223955235407997, 1.3049501874408618, 1.2055297161045848, 1.1692542365736316, 1.0769571962730058, 1.0144505847411098, 0.9913601784485884, 0.9317001386730132, 0.857757082878185, 0.8155634736467603, 0.7682612945963089, 0.7217806418748633, 0.678338533281248, 0.6511820384364709, 0.6192882306865346, 0.6053793146733001, 0.5848721334223365, 0.5718144052517429, 0.5462281883927541, 0.527515083157495, 0.5112993585545178, 0.496920907994256, 0.47722358366421574, 0.46552107689700334, 0.4601234027970099, 0.45317424906679377 ], "peak_layer": 1, "peak_normalized": 2.5461641896458365, "population_cv": 0.5686091737198813, "rest_mean": 9.231269224848668e-05, "spike_contrast": 0.5437781067692293, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.0197621021652594e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00021828828903380781, 0.00019746273756027222, 0.00017939221288543195, 0.0001619200484128669, 0.00014601621660403907, 0.000131807682919316, 0.00012194511509733275, 0.00011187626660102978, 0.00010335272963857278, 0.00010024275252362713, 9.232992306351662e-05, 8.697109296917915e-05, 8.499150135321543e-05, 7.987671415321529e-05, 7.353741239057854e-05, 6.992006092332304e-05, 6.586474046343938e-05, 6.187985127326101e-05, 5.8155463193543255e-05, 5.582727681030519e-05, 5.309295011102222e-05, 5.1900507969548926e-05, 5.0142381951445714e-05, 4.902291402686387e-05, 4.682935104938224e-05, 4.522503513726406e-05, 4.3834825191879645e-05, 4.260212881490588e-05, 4.0913433622336015e-05, 3.991015182691626e-05, 3.944739728467539e-05, 3.885163096128963e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.00021242323236947414, "first_to_last_ratio": 1.8345651540958352, "imbalance_abs_log_ratio": 0.6068074801791611, "last_quartile_mean": 0.0001157894184871111, "mean": 0.00015111673860701558, "normalized": [ 1.771707915810251, 1.6366668487510905, 1.5242909365839221, 1.4186705636795862, 1.3264449439106374, 1.2453857912414694, 1.1893415085703798, 1.1330085309323608, 1.0858463851390703, 1.062861239127963, 1.023237806768965, 0.9942126959777651, 0.978436257513136, 0.9531027445378829, 0.9232414968561398, 0.904013237249231, 0.8853704564278231, 0.8680164030054869, 0.8515402788738432, 0.8395817783841374, 0.8267435189825344, 0.8185143642831039, 0.809172703595369, 0.8007918347797284, 0.7909896391756025, 0.7821052393452608, 0.7753388667989797, 0.7682453290925442, 0.7604995795745633, 0.7549901567959042, 0.7508572350862155, 0.7467737131490535 ], "peak_layer": 1, "peak_normalized": 1.771707915810251, "population_cv": 0.2714170004285523, "rest_mean": 0.00015645500981783562, "spike_contrast": 0.7816309251520459, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012229007406858728, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0002677347220014781, 0.00024732775636948645, 0.00023034587502479553, 0.00021438486874103546, 0.00020044803386554122, 0.0001881986390799284, 0.00017972940986510366, 0.00017121655400842428, 0.00016408956435043365, 0.00016061612404882908, 0.0001546283601783216, 0.00015024218009784818, 0.00014785809617023915, 0.0001440297783119604, 0.00013951724395155907, 0.000136611532070674, 0.0001337942958343774, 0.00013117180787958205, 0.0001286819897359237, 0.00012687486014328897, 0.00012493478425312787, 0.00012369122123345733, 0.00012227953993715346, 0.00012101305037504062, 0.00011953177454415709, 0.00011818919301731512, 0.00011716668086592108, 0.00011609472858253866, 0.00011492421617731452, 0.00011409165017539635, 0.00011346709652571008, 0.00011285000800853595 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.18837904930114746, "first_to_last_ratio": 1.8345651581276443, "imbalance_abs_log_ratio": 0.6068074823768531, "last_quartile_mean": 0.10268321540206671, "mean": 0.13401183625683188, "normalized": [ 1.7717078885231814, 1.6366668119471057, 1.524290957183596, 1.418670514273678, 1.326444920903592, 1.2453858718827995, 1.1893415500975306, 1.1330085716119305, 1.0858463365841577, 1.062861212873729, 1.0232377434953237, 0.9942127340740732, 0.9784362443707608, 0.9531027271661117, 0.9232414917376033, 0.904013241653558, 0.885370477646398, 0.8680164390463408, 0.8515402681847718, 0.8395816969955481, 0.8267435345782925, 0.8185144279720658, 0.8091727254708025, 0.8007918405918669, 0.7909896328038464, 0.7821052665924377, 0.7753388466443417, 0.7682453528580236, 0.7604996016741158, 0.7549901615676585, 0.7508572336421532, 0.7467736753526057 ], "peak_layer": 1, "peak_normalized": 1.7717078885231814, "population_cv": 0.27141699472708714, "rest_mean": 0.1387458680956452, "spike_contrast": 0.7816309473985974, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.10844806432724, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.23742982745170593, 0.2193327248096466, 0.2042730301618576, 0.19011864066123962, 0.1777593195438385, 0.16689644753932953, 0.15938584506511688, 0.151836559176445, 0.1455162614583969, 0.1424359828233719, 0.13712596893310547, 0.13323627412319183, 0.13112203776836395, 0.12772704660892487, 0.1237252876162529, 0.1211484745144844, 0.11865012347698212, 0.11632447689771652, 0.11411647498607635, 0.11251388490200043, 0.11079341918230057, 0.10969062149524689, 0.10843872278928757, 0.10731558501720428, 0.10600197315216064, 0.10481136292219162, 0.10390458256006241, 0.1029539704322815, 0.10191594809293747, 0.10117761790752411, 0.10062375664710999, 0.10007651150226593 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 6.873187476230669e-05, "first_to_last_ratio": 2.248757500145603, "imbalance_abs_log_ratio": 0.8103778415280068, "last_quartile_mean": 3.0564378221242805e-05, "mean": 4.685761763312257e-05, "normalized": [ 1.7400006587426948, 1.6405194725375354, 1.5551306973978087, 1.4944078341288562, 1.4327724019957897, 1.3414187773842097, 1.2804088050749736, 1.2499327782691996, 1.1927955171655855, 1.1608931065609767, 1.0981756972550076, 1.0577423904261343, 1.0542786038595535, 1.0218640087393278, 0.9775807389332813, 0.9570972402588515, 0.9209982861589985, 0.8815092249082567, 0.8478232134430647, 0.8204952153707635, 0.7871780668180614, 0.7759097745574741, 0.7506963524238334, 0.7421155343193898, 0.7072927325438048, 0.6891814936654451, 0.6695399899996352, 0.6616343983211914, 0.6345111343049281, 0.6212687927046295, 0.6200231913288072, 0.6148038704019305 ], "peak_layer": 1, "peak_normalized": 1.7400006587426948, "population_cv": 0.3217536161653951, "rest_mean": 4.900405669845727e-05, "spike_contrast": 0.719671983430414, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.526684668031521e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 8.153228554874659e-05, 7.687083416385576e-05, 7.286971958819777e-05, 7.002439087955281e-05, 6.71363013680093e-05, 6.285568815656006e-05, 5.999690620228648e-05, 5.8568872191244736e-05, 5.5891556257847697e-05, 5.439668530016206e-05, 5.145789691596292e-05, 4.9563288484932855e-05, 4.9400983698433265e-05, 4.7882112994557247e-05, 4.580710447044112e-05, 4.484729652176611e-05, 4.3155785533599555e-05, 4.130542220082134e-05, 3.972697595600039e-05, 3.844645107164979e-05, 3.688528886414133e-05, 3.635728353401646e-05, 3.517584264045581e-05, 3.477376594673842e-05, 3.314205241622403e-05, 3.229340290999971e-05, 3.1373048841487616e-05, 3.10026116494555e-05, 2.9731680115219206e-05, 2.9111175535945222e-05, 2.9052809622953646e-05, 2.8808244678657502e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 0.0001719075116852764, "first_to_last_ratio": 1.4894635014227988, "imbalance_abs_log_ratio": 0.39841598895878805, "last_quartile_mean": 0.00011541572621354135, "mean": 0.00013894635094402474, "normalized": [ 1.3738985779819257, 1.3296082759737424, 1.2883384004275333, 1.2499940655466777, 1.2130730176151365, 1.1755503979222366, 1.147409009170091, 1.11990595257071, 1.0929037170303153, 1.075598370036652, 1.0512762916941902, 1.0312594730951, 1.0194418974669357, 1.0018996501730997, 0.9803460168062212, 0.9648004557293595, 0.9493010807852901, 0.9344788929945884, 0.9198953859218769, 0.908001147597105, 0.8952913731815082, 0.886953676542665, 0.8771894463102521, 0.8683888934041172, 0.8579399877904575, 0.8482799643655919, 0.8408636339472488, 0.8330870307470335, 0.82434015220157, 0.8181187963300758, 0.8135579418994486, 0.809009026741244 ], "peak_layer": 1, "peak_normalized": 1.3738985779819257, "population_cv": 0.16161423472023242, "rest_mean": 0.0001421073115866252, "spike_contrast": 0.8576417505420824, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012187716347398237, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0001908981939777732, 0.0001847442181315273, 0.00017900991952046752, 0.00017368211410939693, 0.00016855206922627985, 0.00016333843814209104, 0.00015942829486448318, 0.00015560684551019222, 0.0001518549834145233, 0.00014945046859793365, 0.00014607100456487387, 0.0001432897406630218, 0.00014164773165248334, 0.00013921030040364712, 0.000136215501697734, 0.0001340555027127266, 0.0001319019211223349, 0.00012984243221580982, 0.0001278161071240902, 0.00012616344611160457, 0.00012439746933523566, 0.00012323897681199014, 0.00012188227265141904, 0.00012065946793882176, 0.00011920763063244522, 0.00011786540562752634, 0.00011683493357850239, 0.00011575440294109285, 0.0001145390560850501, 0.00011367462138878182, 0.00011304090730845928, 0.00011240885214647278 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 0.00014552221000485588, "first_to_last_ratio": 1.262697055715921, "imbalance_abs_log_ratio": 0.2332499537229619, "last_quartile_mean": 0.00011524712863320019, "mean": 0.00012953460168318998, "normalized": [ 1.1676213631634658, 1.1525055688027706, 1.1391580058081985, 1.1289307982409096, 1.1166731577821778, 1.1054001772932092, 1.094813144753513, 1.0822854389705017, 1.0679308603491235, 1.0564894822224282, 1.0456838360716472, 1.0337072702557388, 1.0234945545465333, 1.0158819477934218, 1.0051772952892828, 0.9951467950359763, 0.9866643397779161, 0.9787559418439278, 0.970448736962688, 0.9607908684400951, 0.9509939230158972, 0.9428061355388626, 0.934924980051271, 0.9261034265480621, 0.9170582603044295, 0.9058562786860701, 0.8981778407016658, 0.8907564927732574, 0.8831168120837269, 0.8778216107141017, 0.8745192645108896, 0.8703053916682415 ], "peak_layer": 1, "peak_normalized": 1.1676213631634658, "population_cv": 0.08967236486753827, "rest_mean": 0.00013110875065817877, "spike_contrast": 0.9231588022206524, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012103419721825049, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00015124736819416285, 0.00014928934979252517, 0.000147560378536582, 0.00014623560127802193, 0.00014464781270362437, 0.00014318757166620344, 0.00014181618462316692, 0.00014019341324456036, 0.0001383339986205101, 0.00013685194426216185, 0.00013545223919209093, 0.00013390085950959474, 0.00013257795944809914, 0.00013159186346456409, 0.00013020524056628346, 0.00012890594371128827, 0.00012780717224813998, 0.00012678276107180864, 0.0001257066905964166, 0.0001244556624442339, 0.00012318661902099848, 0.0001221260172314942, 0.00012110513489460573, 0.0001199624384753406, 0.00011879077646881342, 0.00011733973224181682, 0.00011634510883595794, 0.00011538378748809919, 0.0001143941844929941, 0.00011370827269274741, 0.00011328050459269434, 0.0001127346622524783 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00029741139769612346, "first_to_last_ratio": 2.278072415841521, "imbalance_abs_log_ratio": 0.8233296537390045, "last_quartile_mean": 0.00013055396993877366, "mean": 0.00019328556300024502, "normalized": [ 1.889398576482247, 1.780233894901915, 1.6772594716902607, 1.566224958959574, 1.4762232610465842, 1.375571144634086, 1.3083686481730985, 1.2364401527267181, 1.173148897934763, 1.1508864184295666, 1.080457040307468, 1.0318582947152708, 1.0081957004611959, 0.9698994715426978, 0.9066127339785177, 0.8754550048258108, 0.8446146862114313, 0.8115724449181009, 0.7806677558079268, 0.7613480996239201, 0.7403134026279838, 0.7294052762839086, 0.7160186976339193, 0.7062575707090604, 0.6942777320753531, 0.6866005528973866, 0.6829938476675228, 0.6753403086482458, 0.6715967314167978, 0.6686860556999406, 0.6640286432895989, 0.6600445236791291 ], "peak_layer": 1, "peak_normalized": 1.889398576482247, "population_cv": 0.3579154260008552, "rest_mean": 0.00020340604748559632, "spike_contrast": 0.681567477924495, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013863494677934796, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0003651934675872326, 0.0003440935106482357, 0.00032419004128314555, 0.0003027286729775369, 0.0002853326441254467, 0.00026587804313749075, 0.00025288877077400684, 0.00023898603103589267, 0.00022675274522043765, 0.00022244972933549434, 0.0002088367473334074, 0.00019944331143051386, 0.00019486967357806861, 0.00018746756541077048, 0.00017523515271022916, 0.00016921281348913908, 0.0001632518251426518, 0.00015686523693148047, 0.00015089180669747293, 0.00014715759607497603, 0.00014309189282357693, 0.00014098350948188454, 0.00013839607709087431, 0.0001365093921776861, 0.0001341938623227179, 0.00013270997442305088, 0.00013201285037212074, 0.00013053353177383542, 0.0001298099523410201, 0.00012924736074637622, 0.00012834715016651899, 0.00012757707736454904 ] } } }, "post_attention_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "31853ffb07b24cc59e774bb20656840b4358b3080b42516d380f2dc0f46404c8", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.0022265110164880753, 0.0040087271481752396, 0.005379301495850086, 0.006410556845366955, 0.0027253651060163975, 0.004838948138058186, 0.006175771821290255, 0.00742313964292407, 0.0030937574338167906, 0.005150026176124811, 0.007019025273621082, 0.008143599145114422, 0.004255065228790045, 0.0066224560141563416, 0.008129785768687725, 0.010099578648805618, 0.0041999719105660915, 0.00719039561226964, 0.009030314162373543, 0.010736005380749702, 0.005131533369421959, 0.00787449348717928, 0.009912566281855106, 0.012113703414797783, 0.0059690922498703, 0.00876442901790142, 0.010806876234710217, 0.012353328987956047, 0.006502739619463682, 0.009470963850617409, 0.012123330496251583, 0.01384514570236206 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.00015962236011546338, "first_to_last_ratio": 3.7698422272252903, "imbalance_abs_log_ratio": 1.3270331510418396, "last_quartile_mean": 4.2341920561739244e-05, "mean": 8.61727822893954e-05, "normalized": [ 2.542734798705971, 2.3221448837042376, 2.101145461274622, 1.8911858524935854, 1.7004204056817789, 1.5354142426015858, 1.4229001770142031, 1.3028733810328659, 1.2049412416934622, 1.1705258152816296, 1.0773908232649334, 1.0133220774376726, 0.990609360100461, 0.9272534949358001, 0.8550385909364457, 0.8185651575045715, 0.7703972051466339, 0.7173947009730917, 0.6760256695667757, 0.6514523109028573, 0.618419077603279, 0.6053620388084041, 0.5838930841859182, 0.5697043695032278, 0.5457802700901755, 0.5276149005446632, 0.5104701332743111, 0.49517988582787786, 0.47527562953524777, 0.46579059316028215, 0.45896435764104826, 0.45181000957238127 ], "peak_layer": 1, "peak_normalized": 2.542734798705971, "population_cv": 0.5705202801130907, "rest_mean": 9.280119640472414e-05, "spike_contrast": 0.542873885449774, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.03793460666202e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00021911453222855926, 0.00020010568550787866, 0.00018106155039276928, 0.00016296874673571438, 0.00014652995741926134, 0.00013231091725174338, 0.0001226152671733871, 0.00011227222421439365, 0.00010383313929196447, 0.00010086746624438092, 9.284176485380158e-05, 8.732078276807442e-05, 8.536356472177431e-05, 7.99040135461837e-05, 7.368105434579775e-05, 7.053803710732609e-05, 6.638727063545957e-05, 6.181989738252014e-05, 5.825501284562051e-05, 5.613745815935545e-05, 5.329089253791608e-05, 5.216573117650114e-05, 5.0315691623836756e-05, 4.9093010602518916e-05, 4.703140439232811e-05, 4.546604395727627e-05, 4.398863165988587e-05, 4.267102849553339e-05, 4.095582335139625e-05, 4.013847137684934e-05, 3.955023566959426e-05, 3.893372559105046e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.00021442477373057045, "first_to_last_ratio": 1.8431254562763404, "imbalance_abs_log_ratio": 0.6114627481432037, "last_quartile_mean": 0.00011633759004325839, "mean": 0.00015221248349917005, "normalized": [ 1.7740805953138137, 1.6484457246979003, 1.5301434809496548, 1.4217241600875643, 1.3264958046529274, 1.2454577490392524, 1.1897861330408546, 1.1336265611690741, 1.086914920172498, 1.064301358569379, 1.0227209980571423, 0.9941753882589759, 0.9788979908211966, 0.9511107967643203, 0.9228143274453928, 0.9052447611165039, 0.8846530997418666, 0.8652930871753776, 0.8497381552743479, 0.8388656494639406, 0.8256334792987466, 0.8176309647342147, 0.8076951735882674, 0.8000661307359944, 0.7892605467734257, 0.78144613040264, 0.7731219607566042, 0.7664988488799842, 0.7584935618388989, 0.7529639051546407, 0.7485195296370439, 0.7441790263875565 ], "peak_layer": 1, "peak_normalized": 1.7740805953138137, "population_cv": 0.2736411578809574, "rest_mean": 0.00015762286892384981, "spike_contrast": 0.7803207938393821, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012299640220589936, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00027003721334040165, 0.00025091401766985655, 0.00023290693934541196, 0.0002164041652576998, 0.000201909220777452, 0.0001895742170745507, 0.00018110030214302242, 0.00017255211423616856, 0.00016544201935175806, 0.00016199995297938585, 0.00015567090304102749, 0.00015132590488065034, 0.0001490004942752421, 0.00014477093645837158, 0.00014046386058907956, 0.00013778955326415598, 0.0001346552453469485, 0.00013170840975362808, 0.00012934075493831187, 0.00012768582382705063, 0.00012567172234412283, 0.00012445363972801715, 0.00012294128828216344, 0.00012178005272289738, 0.00012013530795229599, 0.00011894585622940212, 0.00011767881369451061, 0.00011667069338727742, 0.00011545218876563013, 0.00011461050598882139, 0.00011393401655368507, 0.00011327333777444437 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.190154030919075, "first_to_last_ratio": 1.84312545379361, "imbalance_abs_log_ratio": 0.6114627467961817, "last_quartile_mean": 0.10316933691501617, "mean": 0.13498355145566165, "normalized": [ 1.774080628234443, 1.6484456620184311, 1.5301434020504485, 1.4217242444591272, 1.326495766996536, 1.2454576834375484, 1.189786122599316, 1.133626627255238, 1.0869149569673349, 1.0643014004698326, 1.0227209907477024, 0.9941753870542451, 0.9788979589698585, 0.9511107512164827, 0.9228143032197887, 0.9052447972745712, 0.8846531331847195, 0.8652931173486506, 0.8497382179112226, 0.8388656682549475, 0.8256334805754935, 0.8176309131099365, 0.8076951535479161, 0.8000661540388709, 0.7892605573866218, 0.7814461538045975, 0.7731219580171443, 0.7664988542586116, 0.7584935821787124, 0.7529638601652359, 0.7485195164462104, 0.7441789968002049 ], "peak_layer": 1, "peak_normalized": 1.774080628234443, "population_cv": 0.27364115345546325, "rest_mean": 0.13978153549962574, "spike_contrast": 0.7803207857775153, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.10907443761825561, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.2394717037677765, 0.22251304984092712, 0.2065441906452179, 0.19190938770771027, 0.1790551096200943, 0.16811630129814148, 0.16060155630111694, 0.1530209481716156, 0.14671564102172852, 0.1436631828546524, 0.13805051147937775, 0.13419732451438904, 0.13213512301445007, 0.1283843070268631, 0.1245647519826889, 0.12219315767288208, 0.11941362172365189, 0.11680033802986145, 0.11470068246126175, 0.1132330670952797, 0.11144693940877914, 0.11036672443151474, 0.10902556031942368, 0.10799577087163925, 0.10653719305992126, 0.10548237711191177, 0.1043587476015091, 0.10346473753452301, 0.10238415747880936, 0.10163773596286774, 0.10103782266378403, 0.10045192390680313 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 6.883367768750759e-05, "first_to_last_ratio": 2.249759357551583, "imbalance_abs_log_ratio": 0.810823258297238, "last_quartile_mean": 3.059601795030176e-05, "mean": 4.691846504556452e-05, "normalized": [ 1.7359059710084281, 1.6475789727938472, 1.5536456441204052, 1.4921194549590373, 1.4365657321885223, 1.3438513058553838, 1.2787343947807137, 1.2483299149152907, 1.1912354282513808, 1.1603083401306369, 1.099547144002948, 1.059972440022979, 1.0583955432739494, 1.0176800955770247, 0.9754963047746087, 0.9631595708780456, 0.9242400656751482, 0.8752301116137658, 0.8454245377074081, 0.8232853322610043, 0.7863494898171388, 0.7774642190168813, 0.7497486129019063, 0.7388483538626245, 0.709315327139865, 0.6908343794807076, 0.6685469983108738, 0.6592365073812129, 0.6314341746588822, 0.6235898169926835, 0.6199474545603838, 0.6139783610863121 ], "peak_layer": 1, "peak_normalized": 1.7359059710084281, "population_cv": 0.3220406003070692, "rest_mean": 4.9070239684725595e-05, "spike_contrast": 0.7193541792517962, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.52988819940947e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 8.144604362314567e-05, 7.730187644483522e-05, 7.28946688468568e-05, 7.000795449130237e-05, 6.740145909134299e-05, 6.305144052021205e-05, 5.999625500408001e-05, 5.8569723478285596e-05, 5.589093780145049e-05, 5.4439886298496276e-05, 5.158906424185261e-05, 4.973227987647988e-05, 4.9658294301480055e-05, 4.774798799189739e-05, 4.576878927764483e-05, 4.51899686595425e-05, 4.3363925215089694e-05, 4.1064453398576006e-05, 3.9666021621087566e-05, 3.86272840842139e-05, 3.689431105158292e-05, 3.6477427784120664e-05, 3.5177054087398574e-05, 3.466563066467643e-05, 3.3279986382694915e-05, 3.2412888685939834e-05, 3.136719897156581e-05, 3.093036502832547e-05, 2.9625922252307646e-05, 2.9257877031341195e-05, 2.9086982976878062e-05, 2.8806922273361124e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 0.00017341253987979144, "first_to_last_ratio": 1.4955148931299893, "imbalance_abs_log_ratio": 0.4024705576674927, "last_quartile_mean": 0.00011595507385209203, "mean": 0.00013990553861731314, "normalized": [ 1.3768818258178377, 1.3340879911950096, 1.2910054180628048, 1.2527236427111486, 1.2147542166907819, 1.1769889670731195, 1.1474411155803916, 1.1220953663134672, 1.0944075691302455, 1.0773007485615413, 1.0508678613623372, 1.0320596054419082, 1.0208161736522385, 1.000559648637946, 0.9805197815181176, 0.9663128275349431, 0.948744191771126, 0.9319305822463463, 0.9182019814348666, 0.9075038868226692, 0.8943650364968407, 0.886262676120959, 0.8757907045344768, 0.8679002174812178, 0.856288373659771, 0.8478178092769747, 0.8387215581135022, 0.8313975747674917, 0.8224084044020752, 0.8161864847722404, 0.8112200992010112, 0.8064376596145924 ], "peak_layer": 1, "peak_normalized": 1.3768818258178377, "population_cv": 0.16325888721442564, "rest_mean": 0.0001431150389921472, "spike_contrast": 0.8564734877369171, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001225742365932092, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0001926333934534341, 0.00018664629897102714, 0.00018061880837194622, 0.0001752629759721458, 0.00016995084297377616, 0.0001646672753849998, 0.00016053336730692536, 0.00015698735660407692, 0.00015311368042603135, 0.00015072034148033708, 0.00014702223415952176, 0.00014439085498452187, 0.0001428178366040811, 0.00013998383656144142, 0.00013718014815822244, 0.00013519251660909504, 0.00013273456715978682, 0.00013038225006312132, 0.00012846154277212918, 0.00012696482008323073, 0.00012512662215158343, 0.0001239930570591241, 0.00012252797023393214, 0.000121424047392793, 0.00011979948612861335, 0.00011861440725624561, 0.00011734179133782163, 0.0001163171255029738, 0.00011505949078127742, 0.00011418900976423174, 0.00011349418491590768, 0.00011282509512966499 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 0.00014675183047074825, "first_to_last_ratio": 1.2676545313577883, "imbalance_abs_log_ratio": 0.23716836728277432, "last_quartile_mean": 0.00011576642282307148, "mean": 0.00013041823922321782, "normalized": [ 1.1700698937843759, 1.1560440994564782, 1.1417398043557874, 1.130463200949706, 1.118403537277814, 1.1065304334184665, 1.0939881912841023, 1.0846815122144253, 1.0709468270677047, 1.0579907390333845, 1.0442838370166418, 1.035550673160169, 1.025959021745254, 1.0140203095307772, 1.0062047700154308, 0.9976631870190885, 0.9861765922450596, 0.9770016877798172, 0.9687702939454803, 0.9602422119882037, 0.9504995944091904, 0.9425883202531964, 0.9330014667281973, 0.9259385264566372, 0.9147443798098084, 0.906202796813466, 0.8958065504175365, 0.8885655303249116, 0.8806194435723184, 0.8757905905296257, 0.8713783728577705, 0.8681336045391749 ], "peak_layer": 1, "peak_normalized": 1.1700698937843759, "population_cv": 0.09121429133712369, "rest_mean": 0.0001320278307604086, "spike_contrast": 0.921975648780332, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001217264449223876, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0001525984553154558, 0.0001507692359155044, 0.000148903694935143, 0.00014743302017450333, 0.00014586022007279098, 0.00014431175077334046, 0.00014267601363826543, 0.00014146225294098258, 0.000139670999487862, 0.00013798128929920495, 0.00013619365927297622, 0.00013505469541996717, 0.00013380376913119107, 0.00013224674330558628, 0.00013122745440341532, 0.00013011347618885338, 0.00012861541472375393, 0.00012741883983835578, 0.0001263453159481287, 0.0001252330985153094, 0.0001239624834852293, 0.00012293070903979242, 0.00012168040848337114, 0.00012075927224941552, 0.00011929935135412961, 0.00011818537313956767, 0.00011682951299007982, 0.00011588515189941972, 0.00011484883725643158, 0.00011421906674513593, 0.00011364363308530301, 0.00011322045611450449 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00030004180916876066, "first_to_last_ratio": 2.2837506327398005, "imbalance_abs_log_ratio": 0.8258191057718037, "last_quartile_mean": 0.00013138116082700435, "mean": 0.00019473525253488333, "normalized": [ 1.9000096511818072, 1.783700741635724, 1.6708066910504589, 1.570366884717585, 1.4783058799917985, 1.376185321596543, 1.3064973297630273, 1.2402700043336092, 1.1767953849206394, 1.1505341943958838, 1.0771542571420398, 1.026098162637436, 1.013898063759585, 0.9669473025647412, 0.9095398902231254, 0.8755205777387188, 0.842301214161826, 0.8073124727024382, 0.7791991818813049, 0.760574607765185, 0.7402055397447304, 0.7308143413325056, 0.7152667122155245, 0.7043714155678553, 0.6942467749765324, 0.6868749151283189, 0.680655564574882, 0.676363713518914, 0.6701061029149729, 0.6672276319278244, 0.6636177363814285, 0.6582317375530352 ], "peak_layer": 1, "peak_normalized": 1.9000096511818072, "population_cv": 0.35930746026536914, "rest_mean": 0.00020494150898971216, "spike_contrast": 0.6812747128050888, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013962146767880766, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0003699988592416048, 0.0003473494143690914, 0.0003253649629186839, 0.0003058057918678969, 0.0002878782688640058, 0.0002679917961359024, 0.0002544210874475539, 0.00024152429250534624, 0.00022916354646440595, 0.000224049566895701, 0.00020975990628357977, 0.00019981748482678086, 0.000197441695490852, 0.0001882987271528691, 0.00017711948021315038, 0.00017049472080543637, 0.00016402573965024203, 0.00015721219824627042, 0.00015173754945863038, 0.00014811068831477314, 0.00014414411270990968, 0.0001423153153154999, 0.0001392876438330859, 0.00013716594548895955, 0.00013519432104658335, 0.00013375876005738974, 0.00013254763325676322, 0.0001317118585575372, 0.00013049328117631376, 0.00012993274140171707, 0.00012922976748086512, 0.00012818092363886535 ] } } }, "post_mlp_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "7fd9c911e93175896ad618842c120e77b3f017c17b7eec9c2bfb01fc6dbc8ac2", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.0034885562490671873, 0.004831995815038681, 0.005952076520770788, 0.006925336085259914, 0.00385080324485898, 0.005542940925806761, 0.006729288958013058, 0.007927131839096546, 0.004081308841705322, 0.005811207927763462, 0.007534808944910765, 0.008581922389566898, 0.00499920267611742, 0.007102519739419222, 0.008542952127754688, 0.010447698645293713, 0.005041283555328846, 0.007636935915797949, 0.009387043304741383, 0.011020516976714134, 0.005807018838822842, 0.008367395959794521, 0.01024632342159748, 0.012378672137856483, 0.006453148555010557, 0.009127611294388771, 0.011037957854568958, 0.012611051090061665, 0.007010402157902718, 0.009794782847166061, 0.012445803731679916, 0.014087481424212456 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.00015858902861509705, "first_to_last_ratio": 3.757869257587208, "imbalance_abs_log_ratio": 1.3238521099622325, "last_quartile_mean": 4.220184837322449e-05, "mean": 8.57325238712292e-05, "normalized": [ 2.5462257183789148, 2.3032637538343064, 2.0924289533379263, 1.8886492391629932, 1.7031256747388024, 1.5374176267930122, 1.4224231993460965, 1.3049597038569594, 1.2055618292025094, 1.169245405308266, 1.0769812048488787, 1.0144334047898542, 0.9913512297333369, 0.9317090592952362, 0.8577499774884012, 0.81555804918997, 0.7682725064221458, 0.7217807351250844, 0.6783324192661732, 0.6511800118936032, 0.6192916242842709, 0.6053728292385717, 0.5848720876072855, 0.5718124973482859, 0.5462302773879167, 0.5275173250132701, 0.5112998770983732, 0.49691833866078555, 0.47721658788927634, 0.46551989467389665, 0.46012635628943105, 0.45317260249616614 ], "peak_layer": 1, "peak_normalized": 2.5462257183789148, "population_cv": 0.5686111305901927, "rest_mean": 9.231303704803361e-05, "spike_contrast": 0.5437775023084314, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.019775271648541e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00021829435718245804, 0.00019746461475733668, 0.0001793892151908949, 0.0001619186659809202, 0.00014601326256524771, 0.00013180669338908046, 0.00012194793089292943, 0.00011187748896190897, 0.00010335585830034688, 0.00010024235962191597, 9.233231685357168e-05, 8.69699360919185e-05, 8.499104296788573e-05, 7.987776916706935e-05, 7.353707042057067e-05, 6.991984992055222e-05, 6.586594099644572e-05, 6.188008410390466e-05, 5.8155150327365845e-05, 5.582730591413565e-05, 5.309343396220356e-05, 5.190014053368941e-05, 5.014256021240726e-05, 4.90229285787791e-05, 4.682970029534772e-05, 4.522539165918715e-05, 4.383502891869284e-05, 4.2602063331287354e-05, 4.0912982512963936e-05, 3.991019548266195e-05, 3.944779382436536e-05, 3.885163096128963e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.00021242368711682502, "first_to_last_ratio": 1.8345658968465297, "imbalance_abs_log_ratio": 0.6068078850437921, "last_quartile_mean": 0.0001157896194854402, "mean": 0.0001511171253696375, "normalized": [ 1.7717451736709768, 1.636674793183293, 1.5242755762025908, 1.4186515254985965, 1.3264266232459345, 1.2453738409548603, 1.1893512719369177, 1.1330135273991528, 1.085860457802012, 1.0628542818802846, 1.023252039669589, 0.9942070699734702, 0.9784308644775869, 0.9531083940389613, 0.9232390376562555, 0.9040072643238835, 0.8853775311228679, 0.8680181295617843, 0.8515367513400655, 0.8395795332993111, 0.8267439067367663, 0.8185133286645813, 0.8091730400239778, 0.800791325997512, 0.7909899739933552, 0.7821072339266707, 0.7753403490721149, 0.7682422073314642, 0.7604971517068638, 0.7549863467424895, 0.7508596466770359, 0.7467718018887739 ], "peak_layer": 1, "peak_normalized": 1.7717451736709768, "population_cv": 0.2714186609524326, "rest_mean": 0.00015645541295978345, "spike_contrast": 0.7816308178118705, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012229037238284946, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0002677410375326872, 0.0002473295899108052, 0.00023034414334688336, 0.00021438254043459892, 0.00020044577831868082, 0.00018819731485564262, 0.00017973134526982903, 0.000171217747265473, 0.0001640921109355986, 0.00016061548376455903, 0.00015463090676348656, 0.00015024171443656087, 0.0001478576596127823, 0.00014403100067283958, 0.00013951722939964384, 0.00013661097909789532, 0.00013379570737015456, 0.0001311724045081064, 0.0001286817860091105, 0.00012687484559137374, 0.0001249351626029238, 0.00012369138130452484, 0.00012227990373503417, 0.00012101328320568427, 0.00011953213106608018, 0.0001181897969217971, 0.0001171672047348693, 0.00011609455395955592, 0.00011492414341773838, 0.0001140913664130494, 0.00011346775136189535, 0.00011285000800853595 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.18837944976985455, "first_to_last_ratio": 1.8345658800697515, "imbalance_abs_log_ratio": 0.6068078758989703, "last_quartile_mean": 0.10268339328467846, "mean": 0.1340121782850474, "normalized": [ 1.7717451751532867, 1.6366747548098712, 1.5242756140248972, 1.4186514377480737, 1.3264266357207033, 1.2453739091726768, 1.1893513017871604, 1.1330134634090196, 1.0858604665419038, 1.0628542749024097, 1.0232520332438115, 0.9942070832344971, 0.9784308561813967, 0.9531084116998004, 0.9232390798282929, 0.9040072094634526, 0.8853775581660035, 0.8680181154203164, 0.8515367049614931, 0.8395794986042229, 0.8267439263823189, 0.8185133396785964, 0.809173050926764, 0.8007913534932437, 0.7909900046670136, 0.782107217826699, 0.7753403703776178, 0.7682422246066585, 0.7604971603471317, 0.7549863443945886, 0.7508596538022533, 0.7467717694238248 ], "peak_layer": 1, "peak_normalized": 1.7717451751532867, "population_cv": 0.27141865758333467, "rest_mean": 0.13874622461972413, "spike_contrast": 0.7816308398662988, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.10844832807779312, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.23743543028831482, 0.21933434903621674, 0.20427149534225464, 0.19011656939983368, 0.17775732278823853, 0.16689527034759521, 0.15938755869865417, 0.15183760225772858, 0.14551852643489838, 0.14243541657924652, 0.13712823390960693, 0.1332358568906784, 0.1311216503381729, 0.12772813439369202, 0.1237252801656723, 0.12114797532558441, 0.1186513751745224, 0.1163249984383583, 0.11411628872156143, 0.11251387745141983, 0.11079375445842743, 0.10969075560569763, 0.10843904316425323, 0.107315793633461, 0.10600229352712631, 0.104811891913414, 0.10390505194664001, 0.10295381397008896, 0.1019158810377121, 0.10117736458778381, 0.10062433779239655, 0.10007651150226593 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 6.873207303215167e-05, "first_to_last_ratio": 2.248757814140914, "imbalance_abs_log_ratio": 0.8103779811585758, "last_quartile_mean": 3.0564462122129044e-05, "mean": 4.6857780716891284e-05, "normalized": [ 1.7400119939228862, 1.6405409364325143, 1.555121247708426, 1.4944054279879166, 1.4327475398601994, 1.3414100714975594, 1.2804101716466747, 1.2499370459040815, 1.192804641985012, 1.1608858830017554, 1.098191983597808, 1.057744066133373, 1.0542717513613309, 1.021869458338179, 0.9775749297615711, 0.9570927445976433, 0.9210091333285576, 0.8815125232807631, 0.847820728512489, 0.8204970180441978, 0.7871838673851995, 0.7759041238100245, 0.7506938949785397, 0.7421070509174572, 0.7072987335050381, 0.6891811912847667, 0.6695407652827761, 0.6616333377893114, 0.6345027924930273, 0.6212629426015038, 0.6200302724102036, 0.614801730639213 ], "peak_layer": 1, "peak_normalized": 1.7400119939228862, "population_cv": 0.32175451551289735, "rest_mean": 4.9004235565748616e-05, "spike_contrast": 0.7196709452950106, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.5266924533061685e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 8.153310045599937e-05, 7.687210745643824e-05, 7.28695304132998e-05, 7.002452184678987e-05, 6.713537004543468e-05, 6.28554989816621e-05, 5.9997179050697014e-05, 5.8569276006892323e-05, 5.589217835222371e-05, 5.439653614303097e-05, 5.1458839152473956e-05, 4.9563539505470544e-05, 4.9400834541302174e-05, 4.788253500009887e-05, 4.58069916930981e-05, 4.4847241952084005e-05, 4.315644400776364e-05, 4.130572051508352e-05, 3.972699778387323e-05, 3.8446669350378215e-05, 3.688568904181011e-05, 3.635714529082179e-05, 3.5175849916413426e-05, 3.477348946034908e-05, 3.3142448955914006e-05, 3.2293501135427505e-05, 3.13731943606399e-05, 3.1002669857116416e-05, 2.9731392714893445e-05, 2.9111002731951885e-05, 2.905324254243169e-05, 2.8808244678657502e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 0.00017190765356644988, "first_to_last_ratio": 1.4894621368061003, "imbalance_abs_log_ratio": 0.3984150727783497, "last_quartile_mean": 0.00011541592721187044, "mean": 0.00013894661265112518, "normalized": [ 1.3739040544627474, 1.3296102750467884, 1.2883342981472405, 1.2499918159027226, 1.2130627732820631, 1.175544413473833, 1.1474083142393565, 1.1199112790671786, 1.092909618037032, 1.0755929927697745, 1.0512861461262244, 1.0312571117882559, 1.0194388253067554, 1.0019061415077835, 0.980344379776871, 0.9647961249933756, 0.9493075664595542, 0.9344817410273759, 0.9198929201794295, 0.9079992279088673, 0.8952920956888009, 0.8869530532606651, 0.8771902552778059, 0.8683889858357474, 0.8579407806493952, 0.8482823463732029, 0.8408658204630546, 0.8330843095877946, 0.8243382853592501, 0.818115422616924, 0.8135611224181157, 0.8090075029660142 ], "peak_layer": 1, "peak_normalized": 1.3739040544627474, "population_cv": 0.16161403917150835, "rest_mean": 0.00014210757001786045, "spike_contrast": 0.8576421569620596, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012187744287075474, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00019089931447524577, 0.00018474484386388212, 0.00017900968668982387, 0.00017368212866131216, 0.0001685509632807225, 0.00016333791427314281, 0.00015942849859129637, 0.00015560787869617343, 0.00015185608936008066, 0.00014945000293664634, 0.00014607264893129468, 0.0001432896824553609, 0.00014164757158141583, 0.0001392114645568654, 0.00013621553080156446, 0.0001340551534667611, 0.00013190307072363794, 0.00012984307250007987, 0.0001278160052606836, 0.00012616341700777411, 0.0001243978040292859, 0.00012323912233114243, 0.00012188261462142691, 0.00012065970804542303, 0.00011920796532649547, 0.00011786595860030502, 0.00011683545744745061, 0.00011575424287002534, 0.00011453901242930442, 0.00011367436673026532, 0.00011304156214464456, 0.00011240885214647278 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 0.00014552358334185556, "first_to_last_ratio": 1.262638843036328, "imbalance_abs_log_ratio": 0.2332038508028458, "last_quartile_mean": 0.00011525352965691127, "mean": 0.00012953498708156985, "normalized": [ 1.1676304712442283, 1.1525836984023852, 1.1391521450612143, 1.1288261090295841, 1.1167600441451297, 1.1054423860149858, 1.094731137320412, 1.082319740349084, 1.067957453000188, 1.0563935463552665, 1.04560972624621, 1.0337959762176818, 1.0235535208863231, 1.0159040893770892, 1.0050628637064216, 0.9950792389307415, 0.9865814183778941, 0.9786230528286792, 0.9704418054120744, 0.9608381133357455, 0.9508407831226554, 0.9429749854457838, 0.9346839821902475, 0.9262276149685831, 0.9170542960948607, 0.9059488475652709, 0.8984648923618161, 0.8908037775288885, 0.8831483920181569, 0.8778579808382188, 0.8744051093280881, 0.8703028022960919 ], "peak_layer": 1, "peak_normalized": 1.1676304712442283, "population_cv": 0.0896642712448568, "rest_mean": 0.00013110964479252557, "spike_contrast": 0.9231344927669946, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012103183544240892, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00015124899800866842, 0.00014929991448298097, 0.00014756005839444697, 0.00014622247545048594, 0.00014465949789155275, 0.00014319346519187093, 0.00014180598373059183, 0.00014019827358424664, 0.00013833785487804562, 0.00013683992438018322, 0.0001354430423816666, 0.00013391274842433631, 0.0001325859921053052, 0.00013159512309357524, 0.00013019080506637692, 0.00012889757636003196, 0.00012779681128449738, 0.00012676592450588942, 0.00012570616672746837, 0.00012446215259842575, 0.00012316714855842292, 0.00012214825255796313, 0.00012107427755836397, 0.00011997888213954866, 0.00011879061639774591, 0.00011735207226593047, 0.0001163826382253319, 0.0001153902558144182, 0.00011439861555118114, 0.00011371332220733166, 0.00011326605454087257, 0.0001127346622524783 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00029742828155576717, "first_to_last_ratio": 2.2781890121807993, "imbalance_abs_log_ratio": 0.8233808344454435, "last_quartile_mean": 0.00013055469935352448, "mean": 0.00019329176166138495, "normalized": [ 1.8891850069388627, 1.7801603926271052, 1.6773544463608039, 1.566500262900107, 1.476456430986117, 1.3754925511314788, 1.3084504582544147, 1.236424592515421, 1.172965600373877, 1.150871569148191, 1.0802294364746856, 1.0319209662753672, 1.0080063247818516, 0.9698985570693867, 0.9067931019504507, 0.8756163462587222, 0.8444435806568995, 0.8113939671049883, 0.7809677247391495, 0.7613442367276676, 0.7402582678385915, 0.7293562882295931, 0.7162426695419845, 0.7062419232587185, 0.6940340549797547, 0.6866100033591334, 0.6831688143117768, 0.6753320518823956, 0.6715919825156708, 0.6686410475337363, 0.6640239865195754, 0.6600233567535221 ], "peak_layer": 1, "peak_normalized": 1.8891850069388627, "population_cv": 0.357915029863326, "rest_mean": 0.00020341356920888993, "spike_contrast": 0.6815376252628051, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001386340009048581, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00036516389809548855, 0.0003440903383307159, 0.0003242187958676368, 0.0003027915954589844, 0.00028538686456158757, 0.00026587137836031616, 0.0002529126941226423, 0.00023899068764876574, 0.00022672458726447076, 0.00022245399304665625, 0.0002087994507746771, 0.00019946182146668434, 0.00019483931828290224, 0.00018747340072877705, 0.00017527563613839447, 0.00016924942610785365, 0.00016322398732881993, 0.00015683576930314302, 0.0001509546273155138, 0.00014716156874783337, 0.0001430858246749267, 0.0001409785618307069, 0.00013844380737282336, 0.00013651074550580233, 0.00013415106514003128, 0.00013271605712361634, 0.00013205090363044292, 0.00013053612201474607, 0.00012981319741811603, 0.00012924280599690974, 0.00012835036613978446, 0.00012757707736454904 ] } } }, "pre_attention_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "de71d4cf2d509a1063f6eac7f50c4885e9a058dee7e4b993ad40f5527f528d00", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.02802751213312149, 0.01413465291261673, 0.014225807972252369, 0.01433455292135477, 0.01440478302538395, 0.009703479707241058, 0.00977881159633398, 0.009858808480203152, 0.009950309991836548, 0.00752287870272994, 0.007571155205368996, 0.007693085819482803, 0.007788818795233965, 0.006353048607707024, 0.006389216054230928, 0.0064707002602517605, 0.00656962301582098, 0.005553483963012695, 0.0056249531917274, 0.005688921548426151, 0.005768724251538515, 0.004966245498508215, 0.005057469941675663, 0.005098348017781973, 0.005216081161051989, 0.0046758754178881645, 0.004727107938379049, 0.004777040798217058, 0.004837613552808762, 0.0044186655431985855, 0.004510571248829365, 0.0045849340967834 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.0001580166108396952, "first_to_last_ratio": 1.9713740353643698, "imbalance_abs_log_ratio": 0.6787307795042399, "last_quartile_mean": 8.015557068574708e-05, "mean": 0.00011843875881822896, "normalized": [ 0.8918679597819287, 1.5814819593588525, 1.5722239987443252, 1.3246067564285042, 1.1252650527356378, 1.512743589128775, 1.3788112428745087, 1.2863035124436846, 1.097095055843533, 1.468154353173571, 1.1523254606068964, 1.0839888571909224, 1.054518217028451, 1.146335080549274, 1.1665208527784754, 0.8931136212023528, 0.8203804970547371, 1.073741223277434, 0.919654742394771, 0.8223971330190447, 0.7711288002244273, 0.8470775332584999, 0.8298711629738247, 0.7662487463744663, 0.7210382462310316, 0.7071376059209309, 0.7189559392079661, 0.727980518763353, 0.6111916361855597, 0.6986829946379066, 0.6244678178312321, 0.6046898327751223 ], "peak_layer": 2, "peak_normalized": 1.5814819593588525, "population_cv": 0.2904715201464359, "rest_mean": 0.00012310891098523926, "spike_contrast": 0.7572151875143335, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.321993711637333e-05, "top_five_layers": [ 2, 3, 6, 10, 7 ], "top_five_spike_overlap": 0 }, "values": [ 0.00010563173418631777, 0.0001873087603598833, 0.00018621225899551064, 0.00015688478015363216, 0.0001332749961875379, 0.00017916747310664505, 0.00016330469225067645, 0.00015234819147735834, 0.00012993857671972364, 0.00017388637934345752, 0.0001364799973089248, 0.0001283862948184833, 0.00012489582877606153, 0.00013577050413005054, 0.00013816128193866462, 0.00010577926877886057, 9.71648478298448e-05, 0.00012717257777694613, 0.00010892276623053476, 9.74036956904456e-05, 9.133153798757121e-05, 0.0001003268116619438, 9.828891052166e-05, 9.07535504666157e-05, 8.539887494407594e-05, 8.375250035896897e-05, 8.515224908478558e-05, 8.622110908618197e-05, 7.238877878990024e-05, 8.275114669231698e-05, 7.396119326585904e-05, 7.161871326388791e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.0001664285200604354, "first_to_last_ratio": 1.9719337364651075, "imbalance_abs_log_ratio": 0.6790146534176558, "last_quartile_mean": 8.439863722742302e-05, "mean": 0.0001245867656507471, "normalized": [ 0.8984334461398115, 1.5927952483854981, 1.5658595485525297, 1.3198735865176374, 1.131213894321221, 1.5125315092450449, 1.3796810696180641, 1.286366052942558, 1.0960606282761096, 1.4536884557404346, 1.1543342746982923, 1.0797559630746076, 1.0497530570909623, 1.1457182987306291, 1.1551962694199982, 0.9023721663305979, 0.8238710779144349, 1.0632503993742801, 0.9211624832492573, 0.8274335806257342, 0.7739032412019898, 0.850990856837025, 0.8281317613098801, 0.7681943949939335, 0.7208251570035091, 0.7089546837555952, 0.7203162530736348, 0.7236174125342815, 0.6145740290254166, 0.6994991973458914, 0.6274381906330746, 0.6042038120380651 ], "peak_layer": 2, "peak_normalized": 1.5927952483854981, "population_cv": 0.2889665774838781, "rest_mean": 0.00012946851159169995, "spike_contrast": 0.7586812913967177, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.822533756960183e-05, "top_five_layers": [ 2, 3, 6, 10, 7 ], "top_five_spike_overlap": 0 }, "values": [ 0.00011193291720701382, 0.0001984412083402276, 0.0001950853766174987, 0.000164438781212084, 0.00014093428035266697, 0.00018844140868168324, 0.00017189000209327787, 0.00016026418597903103, 0.00013655464863404632, 0.00018111034296453, 0.00014381477376446128, 0.0001345233031315729, 0.00013078533811494708, 0.00014274133718572557, 0.00014392216689884663, 0.00011242362961638719, 0.00010264343291055411, 0.0001324669283349067, 0.00011476465442683548, 0.00010308727360097691, 9.641810174798593e-05, 0.00010602219845168293, 0.00010317425767425448, 9.570685506332666e-05, 8.980527491075918e-05, 8.832637104205787e-05, 8.974187221610919e-05, 9.015315299620852e-05, 7.656779052922502e-05, 8.714834257261828e-05, 7.817049481673166e-05, 7.527579873567447e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.14759047608822584, "first_to_last_ratio": 1.9719337947467823, "imbalance_abs_log_ratio": 0.6790146829732503, "last_quartile_mean": 0.07484555337578058, "mean": 0.1104847895912826, "normalized": [ 0.8984334974418968, 1.5927952138130326, 1.565859646322909, 1.3198735575362357, 1.1312139236939245, 1.5125315007911495, 1.3796810264894532, 1.286366010451999, 1.0960606168617568, 1.4536884922319049, 1.1543342013148132, 1.0797558953237962, 1.0497530983161607, 1.1457182562234747, 1.1551962948455465, 0.9023721940039697, 0.8238711061937815, 1.0632504861804335, 0.9211625151546727, 0.8274335809310764, 0.7739032642161865, 0.8509909070635955, 0.8281317391228538, 0.7681943896830757, 0.7208251689669645, 0.7089546589522648, 0.7203162343708721, 0.7236173971219663, 0.6145740294939214, 0.6994991437375198, 0.6274381319743458, 0.6042038211744466 ], "peak_layer": 2, "peak_normalized": 1.5927952138130326, "population_cv": 0.28896658018674404, "rest_mean": 0.11481397047086998, "spike_contrast": 0.7586813040631772, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.08710721284151077, "top_five_layers": [ 2, 3, 6, 10, 7 ], "top_five_spike_overlap": 0 }, "values": [ 0.09926323592662811, 0.1759796440601349, 0.1730036735534668, 0.14582595229148865, 0.12498193234205246, 0.16711172461509705, 0.15243376791477203, 0.14212387800216675, 0.12109802663326263, 0.16061046719551086, 0.1275363713502884, 0.1192966029047966, 0.11598175019025803, 0.12658444046974182, 0.12763161957263947, 0.09969840198755264, 0.0910252258181572, 0.11747300624847412, 0.10177444666624069, 0.09141882508993149, 0.08550453931093216, 0.09402155131101608, 0.09149596095085144, 0.08487379550933838, 0.0796402171254158, 0.0783287063241005, 0.07958398759365082, 0.07994871586561203, 0.06790108233690262, 0.07728401571512222, 0.069322369992733, 0.06675533205270767 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 5.093040158499207e-05, "first_to_last_ratio": 0.8860093039525244, "imbalance_abs_log_ratio": 0.12102782735710801, "last_quartile_mean": 5.748291960117058e-05, "mean": 5.7093861016710434e-05, "normalized": [ 0.4337136137178416, 0.9185569667519601, 0.9444095926066538, 0.8728787927927439, 0.7849140301431445, 1.1296292591692207, 1.0726593944235638, 0.9796135302182263, 0.9386642177229508, 1.3046353648934683, 1.0461832118506482, 1.0032917361864164, 0.9662905330123907, 1.0760080966276753, 1.1828739682986207, 0.9087955578115051, 0.9194941496879511, 1.198292373503509, 1.0946132296674007, 1.0673695485587213, 0.9244260207272295, 1.0525679458300632, 1.125653431968099, 0.9999504891362015, 0.9629027708994021, 1.0063234987710272, 0.9975933861195996, 1.0791525148711678, 0.9412468232151083, 1.085173475571988, 1.0031556955576197, 0.9789667796878816 ], "peak_layer": 10, "peak_normalized": 1.3046353648934683, "population_cv": 0.1441687540611157, "rest_mean": 5.695535414647397e-05, "spike_contrast": 1.0155638391297448, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.784179811598733e-05, "top_five_layers": [ 10, 18, 15, 6, 23 ], "top_five_spike_overlap": 1 }, "values": [ 2.4762384782661684e-05, 5.2443963795667514e-05, 5.3919990023132414e-05, 4.9836020480142906e-05, 4.481377254705876e-05, 6.449489592341706e-05, 6.124226638348773e-05, 5.592991874436848e-05, 5.3591964388033375e-05, 7.448667020071298e-05, 5.973063889541663e-05, 5.728179894504137e-05, 5.516925739357248e-05, 6.143345672171563e-05, 6.75348419463262e-05, 5.1886647270293906e-05, 5.249747118796222e-05, 6.84151382301934e-05, 6.249569560168311e-05, 6.094024865888059e-05, 5.277905074763112e-05, 6.009516800986603e-05, 6.426790059776977e-05, 5.7091034250333905e-05, 5.4975836974335834e-05, 5.74548939766828e-05, 5.695645813830197e-05, 6.161298369988799e-05, 5.373941530706361e-05, 6.19567435933277e-05, 5.727403186028823e-05, 5.58929932594765e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 5.801741053801379e-05, "first_to_last_ratio": 0.954195857595535, "imbalance_abs_log_ratio": 0.046886327142486896, "last_quartile_mean": 6.08024129178375e-05, "mean": 6.296396202287724e-05, "normalized": [ 0.4796364558374551, 0.9568353419130547, 0.9804638447804884, 0.8949545354119591, 0.8237377234766136, 1.1597846916175047, 1.085649359633547, 0.9904450076993521, 0.9595078978373608, 1.2941748206155441, 1.0680753760655664, 1.0231559828609242, 0.9904019047543832, 1.096490963646882, 1.1850351594596897, 0.9422215946487361, 0.9418631353047864, 1.1705024179981143, 1.0926353878857977, 1.0382275689133567, 0.9320833313178482, 1.0784583323883348, 1.0985080196909318, 0.9917906747078287, 0.9476388143731139, 0.9855066582268529, 0.978396290111904, 1.028407839812513, 0.8982055143685478, 1.0282804954549911, 0.9451124964564701, 0.9138123627295472 ], "peak_layer": 10, "peak_normalized": 1.2941748206155441, "population_cv": 0.13431233496292436, "rest_mean": 6.285090866084729e-05, "spike_contrast": 1.0115120295379672, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.357445017783903e-05, "top_five_layers": [ 10, 15, 18, 6, 23 ], "top_five_spike_overlap": 1 }, "values": [ 3.019981159013696e-05, 6.0246144130360335e-05, 6.173388828756288e-05, 5.634988337988034e-05, 5.186579073779285e-05, 7.302463927771896e-05, 6.835678505012766e-05, 6.236234185053036e-05, 6.041441884008236e-05, 8.148637425620109e-05, 6.725025741616264e-05, 6.442195444833487e-05, 6.235962791834027e-05, 6.903941539349034e-05, 7.461450877599418e-05, 5.9326004702597857e-05, 5.930343468207866e-05, 7.369946979451925e-05, 6.879665306769311e-05, 6.537092122016475e-05, 5.8687659475253895e-05, 6.790400948375463e-05, 6.916641723364592e-05, 6.244707037694752e-05, 5.966709431959316e-05, 6.205140380188823e-05, 6.160370685392991e-05, 6.475263216998428e-05, 5.655457789544016e-05, 6.474461406469345e-05, 5.950802733423188e-05, 5.753724690293893e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 1.7888660295284353e-05, "first_to_last_ratio": 0.41855462748958633, "imbalance_abs_log_ratio": 0.8709478659353102, "last_quartile_mean": 4.2739129185065394e-05, "mean": 3.1986895351110434e-05, "normalized": [ 0.2593955501951102, 0.4997829486650695, 0.48480994473028793, 0.4968760363401999, 0.5474125689219601, 0.7788406177150305, 0.7554473604850604, 0.6514322393717875, 0.7578135841604356, 0.9674810188795085, 0.9032772556875033, 0.9019220651946442, 0.8311356963419073, 0.9587970177975769, 1.1401394869948454, 0.8953886492354074, 1.0262272842074007, 1.1413916918814533, 1.174037958462901, 1.194927715786966, 1.0267140632187253, 1.3434898267078448, 1.3729067467361273, 1.2011929483563188, 1.1735959904166378, 1.3266745689353452, 1.2739952988364465, 1.4049535244931144, 1.2752889026856997, 1.4680979746684197, 1.4182292846928506, 1.3483241791974143 ], "peak_layer": 30, "peak_normalized": 1.4680979746684197, "population_cv": 0.3184955307507873, "rest_mean": 3.066251991657299e-05, "spike_contrast": 1.276428773763581, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.913852269761264e-05, "top_five_layers": [ 30, 31, 28, 23, 32 ], "top_five_spike_overlap": 1 }, "values": [ 8.297258318634704e-06, 1.5986504877218977e-05, 1.5507564967265353e-05, 1.589352177688852e-05, 1.7510028555989265e-05, 2.491269333404489e-05, 2.416441566310823e-05, 2.083729486912489e-05, 2.4240103812189773e-05, 3.094671410508454e-05, 2.889303505071439e-05, 2.8849686714238487e-05, 2.6585450541460887e-05, 3.066893987124786e-05, 3.6469522456172854e-05, 2.8640703021665104e-05, 3.282582474639639e-05, 3.6509576602838933e-05, 3.755382931558415e-05, 3.822202779701911e-05, 3.2841395295690745e-05, 4.2974068492185324e-05, 4.391502443468198e-05, 3.842243313556537e-05, 3.7539692129939795e-05, 4.243620060151443e-05, 4.0751154301688075e-05, 4.494010136113502e-05, 4.0792532672639936e-05, 4.695989628089592e-05, 4.5364751713350415e-05, 4.3128704419359565e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00022091567916504573, "first_to_last_ratio": 1.232986382535498, "imbalance_abs_log_ratio": 0.20943917994887448, "last_quartile_mean": 0.00017917122386279516, "mean": 0.0002198539991695725, "normalized": [ 0.4672441326905094, 1.0299890566632133, 1.08986441571188, 1.0084939820045402, 0.8814237539069011, 1.2693826047962031, 1.1886514905905403, 1.1035827467935222, 1.0473307057033197, 1.4274044616309502, 1.1442998336482333, 1.0911283607583675, 1.1094038041384326, 1.1946873974128753, 1.2511466100570947, 0.9923165311956169, 0.9239703707694655, 1.2473578196113966, 1.1181336030529359, 1.0160745414007064, 0.9397394354798955, 0.9905446518023159, 1.0121913654482901, 0.9359943959598221, 0.8839446941554127, 0.8510229499852975, 0.8648513533654478, 0.8786898836618169, 0.7455559493639876, 0.8399702482950041, 0.7392957942269532, 0.7163130557190521 ], "peak_layer": 10, "peak_normalized": 1.4274044616309502, "population_cv": 0.18952709251819305, "rest_mean": 0.0002217885959436427, "spike_contrast": 0.9441746799407325, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00020940717658959329, "top_five_layers": [ 10, 6, 15, 18, 14 ], "top_five_spike_overlap": 0 }, "values": [ 0.00010272549116052687, 0.00022644721320830286, 0.00023961105034686625, 0.00022172143508214504, 0.0001937845372594893, 0.0002790788421407342, 0.0002613297838252038, 0.00024262708029709756, 0.00023025984410196543, 0.000313820579322055, 0.00025157889467664063, 0.00023988893372006714, 0.00024390686303377151, 0.000262656802078709, 0.00027506958576850593, 0.0002181647578254342, 0.00020313858112785965, 0.00027423660503700376, 0.0002458261442370713, 0.00022338805138133466, 0.00020660547306761146, 0.00021777520305477083, 0.00022253431961871684, 0.00020578211115207523, 0.0001943387760547921, 0.00018710079893935472, 0.0001901410287246108, 0.00019318348495289683, 0.00016391345707233995, 0.00018467081827111542, 0.00016253713693004102, 0.00015748428995721042 ] } } }, "pre_mlp_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "2999a257265024350fd2665e2a8d452a03cd662193b4245d15290408ad29922d", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.014061377383768559, 0.014165290631353855, 0.014281988143920898, 0.014350161887705326, 0.00966146495193243, 0.009741888381540775, 0.009820535778999329, 0.00991273857653141, 0.0074927168898284435, 0.007546746172010899, 0.00766373984515667, 0.007762390188872814, 0.006325984839349985, 0.006375366821885109, 0.006445344537496567, 0.006555322092026472, 0.005531130358576775, 0.005611324217170477, 0.0056777093559503555, 0.00574630219489336, 0.004946626722812653, 0.0050392793491482735, 0.005084683652967215, 0.005203457549214363, 0.004667261149734259, 0.004717131145298481, 0.004768412560224533, 0.004826710093766451, 0.0044052014127373695, 0.004504534415900707, 0.004574235528707504, 0.004644096828997135 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 3.729895206561196e-05, "first_to_last_ratio": 1.2469721805352876, "imbalance_abs_log_ratio": 0.22071835733540474, "last_quartile_mean": 2.9911615229138988e-05, "mean": 3.340667609563752e-05, "normalized": [ 1.308752778077389, 1.2456464596610868, 1.0236956931411572, 0.9922285647386262, 1.26903181062375, 1.1170846373362797, 1.0287316533628064, 0.9469237624012169, 1.1300261751959062, 1.1132698789554223, 1.0126240729566007, 0.9379202574439188, 1.1130503370514597, 1.070713488608195, 0.9897642722064988, 0.922085688720865, 1.036831181462568, 0.9898105546019274, 0.9395130252451869, 0.9001967292772246, 0.9975952535130206, 0.9404117749145336, 0.9224862220158919, 0.8885787590271026, 0.9461670180110204, 0.8974372095121397, 0.8904691498559152, 0.8684437934704528, 0.9244147277585869, 0.9015689206268677, 0.8740901912628669, 0.8604359589635168 ], "peak_layer": 1, "peak_normalized": 1.308752778077389, "population_cv": 0.1164905978879517, "rest_mean": 3.378375206213807e-05, "spike_contrast": 0.9285666618330386, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.137046587653458e-05, "top_five_layers": [ 1, 5, 2, 9, 6 ], "top_five_spike_overlap": 0 }, "values": [ 4.3721080146497115e-05, 4.161290780757554e-05, 3.419827044126578e-05, 3.314705827506259e-05, 4.239413465256803e-05, 3.73180846509058e-05, 3.4366505133220926e-05, 3.1633575417799875e-05, 3.7750418414361775e-05, 3.719064625329338e-05, 3.3828404411906376e-05, 3.1332798243965954e-05, 3.718331208801828e-05, 3.5768978705164045e-05, 3.306473445263691e-05, 3.080381793552078e-05, 3.463708344497718e-05, 3.3066280593629926e-05, 3.138600732199848e-05, 3.007258055731654e-05, 3.332634150865488e-05, 3.1416031561093405e-05, 3.0817198421573266e-05, 2.9684462788281962e-05, 3.1608295103069395e-05, 2.9980394174344838e-05, 2.9747614462394267e-05, 2.9011820515734144e-05, 3.0881623388268054e-05, 3.01184209092753e-05, 2.9200447897892445e-05, 2.874430538213346e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 6.705871965095866e-05, "first_to_last_ratio": 0.7180297289018083, "imbalance_abs_log_ratio": 0.33124430563536084, "last_quartile_mean": 9.33926785364747e-05, "mean": 8.228966771639534e-05, "normalized": [ 0.8685826408392191, 0.8249006353131848, 0.7154905339803773, 0.6884730939760333, 0.9470487095509537, 0.8638069620260335, 0.8256155016985178, 0.7853664463613442, 0.9851686390305121, 0.9692511243547932, 0.9083804947955202, 0.8729855457896263, 1.0422548534804916, 1.0255492616133721, 0.9871528463964067, 0.9398223255303129, 1.0958375595009802, 1.0726325633807163, 1.0495259774381736, 1.0230838788771643, 1.1576441025653552, 1.1142532603054807, 1.1008567438213033, 1.056908774400763, 1.159063578749664, 1.1192434432436902, 1.1032241584740294, 1.1060435702772593, 1.1786235078649367, 1.1628365878275773, 1.141840825041133, 1.1085318534950752 ], "peak_layer": 29, "peak_normalized": 1.1786235078649367, "population_cv": 0.1341844256762087, "rest_mean": 8.049536754023629e-05, "spike_contrast": 1.1426606459269575, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.197888866765424e-05, "top_five_layers": [ 29, 30, 25, 21, 31 ], "top_five_spike_overlap": 2 }, "values": [ 7.14753768988885e-05, 6.788079917896539e-05, 5.887747829547152e-05, 5.66542221349664e-05, 7.793232362018898e-05, 7.108238787623122e-05, 6.793962529627606e-05, 6.462754390668124e-05, 8.106919995043427e-05, 7.975935295689851e-05, 7.475032907677814e-05, 7.183769048424438e-05, 8.576680556870997e-05, 8.4392107964959e-05, 8.123247971525416e-05, 7.733766688033938e-05, 9.017610864248127e-05, 8.826657722238451e-05, 8.636514394311234e-05, 8.418923243880272e-05, 9.526214853394777e-05, 9.169153054244816e-05, 9.0589135652408e-05, 8.697267185198143e-05, 9.537895675748587e-05, 9.210217103827745e-05, 9.078394941752777e-05, 9.101595787797123e-05, 9.698853682493791e-05, 9.56894364207983e-05, 9.396170207764953e-05, 9.122071787714958e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.05946834245696664, "first_to_last_ratio": 0.7180297222365345, "imbalance_abs_log_ratio": 0.33124431491808765, "last_quartile_mean": 0.08282156102359295, "mean": 0.07297530013602227, "normalized": [ 0.8685826122359729, 0.8249005917343738, 0.71549052911611, 0.6884731350635278, 0.9470486699630107, 0.8638070113577097, 0.8256154769796886, 0.7853664269052657, 0.9851686341386658, 0.9692511556795396, 0.9083805454214465, 0.8729855579949275, 1.0422548040385637, 1.0255492365750694, 0.9871528950273681, 0.9398223256645052, 1.0958375053304956, 1.07263252629438, 1.0495259690486791, 1.0230837911446407, 1.1576441435704576, 1.1142533045483842, 1.1008568124470575, 1.0569088284972943, 1.1590636022344982, 1.119243411839159, 1.1032241423789504, 1.1060435591240758, 1.1786235037173105, 1.1628366076958818, 1.141840809884862, 1.1085318743481278 ], "peak_layer": 29, "peak_normalized": 1.1786235037173105, "population_cv": 0.13418442668798974, "rest_mean": 0.07138409617322462, "spike_contrast": 1.1426607032635474, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.08156780153512955, "top_five_layers": [ 29, 30, 25, 21, 31 ], "top_five_spike_overlap": 2 }, "values": [ 0.06338507682085037, 0.0601973682641983, 0.05221313610672951, 0.050241533666849136, 0.0691111609339714, 0.06303657591342926, 0.060249537229537964, 0.05731235072016716, 0.07189297676086426, 0.07073139399290085, 0.06628934293985367, 0.06370638310909271, 0.07605885714292526, 0.0748397633433342, 0.07203777879476547, 0.06858381628990173, 0.07996907085180283, 0.07827568054199219, 0.07658947259187698, 0.07465984672307968, 0.0844794288277626, 0.08131296932697296, 0.0803353562951088, 0.07712823897600174, 0.08458301424980164, 0.08167712390422821, 0.08050811290740967, 0.08071386069059372, 0.0860104039311409, 0.08485835045576096, 0.08332617580890656, 0.08089544624090195 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 7.408341275549901e-06, "first_to_last_ratio": 0.49124437919492936, "imbalance_abs_log_ratio": 0.7108135577065074, "last_quartile_mean": 1.5080765479069669e-05, "mean": 1.1844782648040564e-05, "normalized": [ 0.5657990507718705, 0.5382233871697658, 0.49942145823009426, 0.48906984415931193, 0.8062921756393574, 0.7122953827323945, 0.7013508394120195, 0.6911626993599347, 0.8940675881372303, 0.868722127857376, 0.9735911331661544, 0.854855860776803, 0.989483587379368, 1.0059088486455374, 1.0303729006301217, 0.9941025538400318, 1.1620912657632405, 1.127012997009919, 1.113401270263767, 1.066097770938846, 1.1826034542812638, 1.1643547935214302, 1.2049778184223279, 1.1791491538213539, 1.2365172470752577, 1.219921447624247, 1.2337761202582145, 1.2288278254433027, 1.3039217543779833, 1.3631114810840155, 1.2856635723507386, 1.3138525898567213 ], "peak_layer": 30, "peak_normalized": 1.3631114810840155, "population_cv": 0.2549101540724733, "rest_mean": 1.1420299651271543e-05, "spike_contrast": 1.237882653019464, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.4136990830593277e-05, "top_five_layers": [ 30, 32, 29, 31, 25 ], "top_five_spike_overlap": 1 }, "values": [ 6.701766778860474e-06, 6.3751390371180605e-06, 5.915538622502936e-06, 5.7929260037781205e-06, 9.550355571263935e-06, 8.436983989668079e-06, 8.307348252856173e-06, 8.186671948351432e-06, 1.0590036254143342e-05, 1.0289824786013924e-05, 1.1531975360412616e-05, 1.0125581866304856e-05, 1.1720218026312068e-05, 1.1914771675947122e-05, 1.220454305439489e-05, 1.1774928680097219e-05, 1.3764718460151926e-05, 1.334922399109928e-05, 1.3187996046326589e-05, 1.2627696378331166e-05, 1.4007680874783546e-05, 1.379152945446549e-05, 1.4272700354922563e-05, 1.3966765436634887e-05, 1.4646278032159898e-05, 1.4449704394792207e-05, 1.4613809980801307e-05, 1.4555198504240252e-05, 1.5444669770658948e-05, 1.614575921848882e-05, 1.5228405572997872e-05, 1.5562298358418047e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 5.6029601637419546e-05, "first_to_last_ratio": 0.6070213107493427, "imbalance_abs_log_ratio": 0.49919138022039955, "last_quartile_mean": 9.230252817360451e-05, "mean": 7.840498903988191e-05, "normalized": [ 0.6598133090619259, 0.6419677920829977, 0.6005256373772865, 0.5913463573822231, 0.8562376513989928, 0.8062968559250673, 0.7923156566726156, 0.768439227921366, 0.9790122729268586, 0.9665730330981341, 0.9163710065350301, 0.8881686280668529, 1.0631190324760253, 1.0511030519715439, 1.0188080221730096, 0.9712812246823179, 1.1361272291219966, 1.1138561419734825, 1.0918134348286874, 1.0639217496772315, 1.203414696914147, 1.1592426077267617, 1.1442664103435203, 1.097952968622282, 1.2042281788778202, 1.1625445130141736, 1.1452986211445626, 1.1473932954411665, 1.2233942816523278, 1.205318760675343, 1.1829469858778916, 1.146901364356358 ], "peak_layer": 29, "peak_normalized": 1.2233942816523278, "population_cv": 0.1898597249088972, "rest_mean": 7.605543874820728e-05, "spike_contrast": 1.1977126437531995, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.109256061492487e-05, "top_five_layers": [ 29, 30, 25, 21, 31 ], "top_five_spike_overlap": 2 }, "values": [ 5.173265526536852e-05, 5.033347770222463e-05, 4.708420601673424e-05, 4.636450466932729e-05, 6.713330367347226e-05, 6.321769615169615e-05, 6.212150037754327e-05, 6.024946924299002e-05, 7.675944652874023e-05, 7.578414806630462e-05, 7.184805872384459e-05, 6.963685154914856e-05, 8.335383608937263e-05, 8.241172326961532e-05, 7.987963181221858e-05, 7.615329377586022e-05, 8.907804294722155e-05, 8.733187860343605e-05, 8.560362039133906e-05, 8.341677312273532e-05, 9.435371612198651e-05, 9.089040395338088e-05, 8.971619536168873e-05, 8.608499047113582e-05, 9.441749716643244e-05, 9.114928980125114e-05, 8.97971258382313e-05, 8.996135875349864e-05, 9.592021524440497e-05, 9.450300422031432e-05, 9.274894546251744e-05, 8.992278890218586e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 4.8511469685763586e-05, "first_to_last_ratio": 0.5271352332510719, "imbalance_abs_log_ratio": 0.6402981537748061, "last_quartile_mean": 9.202850924339145e-05, "mean": 7.532146378252946e-05, "normalized": [ 0.548104082267993, 0.5416830199005969, 0.528884702689083, 0.5247448675895723, 0.7777813723908718, 0.7487613756360456, 0.7496059866168319, 0.7329069161583202, 0.9480676391902623, 0.9399123881532756, 0.9000936195631334, 0.883441930780847, 1.0560589316963704, 1.0534552095740957, 1.032095781768693, 0.9835620691098637, 1.1601754560133901, 1.1467788545782247, 1.1233340534265408, 1.0967375255670042, 1.2396083970871608, 1.1951467072205837, 1.1828066060767042, 1.1317734013234224, 1.244698956870482, 1.2025862582796845, 1.1881970083126314, 1.1882256015362047, 1.2689460104605979, 1.2589162611011824, 1.2307315535498626, 1.1921774555104672 ], "peak_layer": 29, "peak_normalized": 1.2689460104605979, "population_cv": 0.23486334953575583, "rest_mean": 7.254842300123225e-05, "spike_contrast": 1.244629176846487, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.029588400153444e-05, "top_five_layers": [ 29, 30, 25, 21, 31 ], "top_five_spike_overlap": 2 }, "values": [ 4.1284001781605184e-05, 4.080035796505399e-05, 3.9836369978729635e-05, 3.952455153921619e-05, 5.858363147126511e-05, 5.639780283672735e-05, 5.6461420172126964e-05, 5.520362174138427e-05, 7.140984234865755e-05, 7.079557690303773e-05, 6.77963689668104e-05, 6.654213939327747e-05, 7.954390457598493e-05, 7.934778841445222e-05, 7.773896504659206e-05, 7.408333476632833e-05, 8.738611359149218e-05, 8.637706196168438e-05, 8.461116522084922e-05, 8.26078758109361e-05, 9.336911898571998e-05, 9.002019942272455e-05, 8.909072494134307e-05, 8.524682925781235e-05, 9.375254740007222e-05, 9.058055729838088e-05, 8.949673792812973e-05, 8.949889161158353e-05, 9.557887096889317e-05, 9.482341556577012e-05, 9.270050213672221e-05, 8.979655103757977e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 9.61669284151867e-05, "first_to_last_ratio": 0.8297833624837898, "imbalance_abs_log_ratio": 0.18659062131596024, "last_quartile_mean": 0.00011589401856326731, "mean": 0.00010716335873439675, "normalized": [ 0.9380742209455786, 0.8898820858744699, 0.7837417114644826, 0.7583102664636657, 1.0822070619439073, 0.9520749081713696, 0.9107982467806989, 0.8640017807748708, 1.0669538303069324, 1.0266950649381863, 0.9594029855154438, 0.8901506143407661, 1.0795289063552718, 1.0379165674077444, 0.9706971366471856, 0.914979822300863, 1.052939971267624, 1.0225707935964177, 0.9851238030379684, 0.9636652893153618, 1.0990044510799908, 1.0523904893769196, 1.0422234788441622, 1.0049019599938585, 1.0962247910521359, 1.0599378706264646, 1.05039930496957, 1.0583562345651827, 1.1199788667078585, 1.1100529546792268, 1.0850973923636869, 1.0717171382921347 ], "peak_layer": 29, "peak_normalized": 1.1199788667078585, "population_cv": 0.09156717959150794, "rest_mean": 0.00010599351123716958, "spike_contrast": 1.0706366257223145, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00011348053521942347, "top_five_layers": [ 29, 30, 21, 25, 31 ], "top_five_spike_overlap": 2 }, "values": [ 0.00010052718425868079, 9.536275319987908e-05, 8.398839418077841e-05, 8.12630751170218e-05, 0.00011597294360399246, 0.00010202754492638633, 9.760419925441965e-05, 9.25893327803351e-05, 0.00011433835607022047, 0.00011002409155480564, 0.00010281284630764276, 9.539152961224318e-05, 0.000115685943455901, 0.0001112266254494898, 0.0001040231654769741, 9.805231093196198e-05, 0.00011283658386673778, 0.00010958212078548968, 0.00010556917550275102, 0.00010326960909878835, 0.00011777300824178383, 0.0001127776995417662, 0.00011168816854478791, 0.00010768866923172027, 0.00011747513053705916, 0.00011358650226611644, 0.00011256431753281504, 0.00011341700883349404, 0.00012002069706795737, 0.00011895700299646705, 0.00011628268111962825, 0.00011484880815260112 ] } } } }, "selector": { "expected_uniform_count": 4, "identities_unique": true, "identity_order_sha256": "97fd9721a263024f6a746638f32e0ad9a988cd4c111c00e9e178c1022e5c9d5e", "passed": true, "uniform_count": 4, "uniform_indices": [ 41, 43, 45, 47 ], "visit_count": 65, "visits": [ { "branch": "attention", "coefficient": "detached_learned", "group": 1, "index": 0, "kind": "depth", "layer": 1, "offset": 1, "sources": 1, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 1, "index": 1, "kind": "depth", "layer": 1, "offset": 1, "sources": 2, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 1, "index": 2, "kind": "depth", "layer": 2, "offset": 2, "sources": 2, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 1, "index": 3, "kind": "depth", "layer": 2, "offset": 2, "sources": 2, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 1, "index": 4, "kind": "depth", "layer": 3, "offset": 3, "sources": 2, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 1, "index": 5, "kind": "depth", "layer": 3, "offset": 3, "sources": 2, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 1, "index": 6, "kind": "depth", "layer": 4, "offset": 4, "sources": 2, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 1, "index": 7, "kind": "depth", "layer": 4, "offset": 4, "sources": 2, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 2, "index": 8, "kind": "depth", "layer": 5, "offset": 1, "sources": 2, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 2, "index": 9, "kind": "depth", "layer": 5, "offset": 1, "sources": 3, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 2, "index": 10, "kind": "depth", "layer": 6, "offset": 2, "sources": 3, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 2, "index": 11, "kind": "depth", "layer": 6, "offset": 2, "sources": 3, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 2, "index": 12, "kind": "depth", "layer": 7, "offset": 3, "sources": 3, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 2, "index": 13, "kind": "depth", "layer": 7, "offset": 3, "sources": 3, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 2, "index": 14, "kind": "depth", "layer": 8, "offset": 4, "sources": 3, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 2, "index": 15, "kind": "depth", "layer": 8, "offset": 4, "sources": 3, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 3, "index": 16, "kind": "depth", "layer": 9, "offset": 1, "sources": 3, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 3, "index": 17, "kind": "depth", "layer": 9, "offset": 1, "sources": 4, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 3, "index": 18, "kind": "depth", "layer": 10, "offset": 2, "sources": 4, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 3, "index": 19, "kind": "depth", "layer": 10, "offset": 2, "sources": 4, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 3, "index": 20, "kind": "depth", "layer": 11, "offset": 3, "sources": 4, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 3, "index": 21, "kind": "depth", "layer": 11, "offset": 3, "sources": 4, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 3, "index": 22, "kind": "depth", "layer": 12, "offset": 4, "sources": 4, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 3, "index": 23, "kind": "depth", "layer": 12, "offset": 4, "sources": 4, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 4, "index": 24, "kind": "depth", "layer": 13, "offset": 1, "sources": 4, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 4, "index": 25, "kind": "depth", "layer": 13, "offset": 1, "sources": 5, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 4, "index": 26, "kind": "depth", "layer": 14, "offset": 2, "sources": 5, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 4, "index": 27, "kind": "depth", "layer": 14, "offset": 2, "sources": 5, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 4, "index": 28, "kind": "depth", "layer": 15, "offset": 3, "sources": 5, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 4, "index": 29, "kind": "depth", "layer": 15, "offset": 3, "sources": 5, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 4, "index": 30, "kind": "depth", "layer": 16, "offset": 4, "sources": 5, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 4, "index": 31, "kind": "depth", "layer": 16, "offset": 4, "sources": 5, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 5, "index": 32, "kind": "depth", "layer": 17, "offset": 1, "sources": 5, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 5, "index": 33, "kind": "depth", "layer": 17, "offset": 1, "sources": 6, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 5, "index": 34, "kind": "depth", "layer": 18, "offset": 2, "sources": 6, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 5, "index": 35, "kind": "depth", "layer": 18, "offset": 2, "sources": 6, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 5, "index": 36, "kind": "depth", "layer": 19, "offset": 3, "sources": 6, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 5, "index": 37, "kind": "depth", "layer": 19, "offset": 3, "sources": 6, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 5, "index": 38, "kind": "depth", "layer": 20, "offset": 4, "sources": 6, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 5, "index": 39, "kind": "depth", "layer": 20, "offset": 4, "sources": 6, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 6, "index": 40, "kind": "depth", "layer": 21, "offset": 1, "sources": 6, "use_uniform": false }, { "branch": "mlp", "coefficient": "uniform", "group": 6, "index": 41, "kind": "depth", "layer": 21, "offset": 1, "sources": 7, "use_uniform": true }, { "branch": "attention", "coefficient": "detached_learned", "group": 6, "index": 42, "kind": "depth", "layer": 22, "offset": 2, "sources": 7, "use_uniform": false }, { "branch": "mlp", "coefficient": "uniform", "group": 6, "index": 43, "kind": "depth", "layer": 22, "offset": 2, "sources": 7, "use_uniform": true }, { "branch": "attention", "coefficient": "detached_learned", "group": 6, "index": 44, "kind": "depth", "layer": 23, "offset": 3, "sources": 7, "use_uniform": false }, { "branch": "mlp", "coefficient": "uniform", "group": 6, "index": 45, "kind": "depth", "layer": 23, "offset": 3, "sources": 7, "use_uniform": true }, { "branch": "attention", "coefficient": "detached_learned", "group": 6, "index": 46, "kind": "depth", "layer": 24, "offset": 4, "sources": 7, "use_uniform": false }, { "branch": "mlp", "coefficient": "uniform", "group": 6, "index": 47, "kind": "depth", "layer": 24, "offset": 4, "sources": 7, "use_uniform": true }, { "branch": "attention", "coefficient": "detached_learned", "group": 7, "index": 48, "kind": "depth", "layer": 25, "offset": 1, "sources": 7, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 7, "index": 49, "kind": "depth", "layer": 25, "offset": 1, "sources": 8, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 7, "index": 50, "kind": "depth", "layer": 26, "offset": 2, "sources": 8, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 7, "index": 51, "kind": "depth", "layer": 26, "offset": 2, "sources": 8, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 7, "index": 52, "kind": "depth", "layer": 27, "offset": 3, "sources": 8, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 7, "index": 53, "kind": "depth", "layer": 27, "offset": 3, "sources": 8, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 7, "index": 54, "kind": "depth", "layer": 28, "offset": 4, "sources": 8, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 7, "index": 55, "kind": "depth", "layer": 28, "offset": 4, "sources": 8, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 8, "index": 56, "kind": "depth", "layer": 29, "offset": 1, "sources": 8, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 8, "index": 57, "kind": "depth", "layer": 29, "offset": 1, "sources": 9, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 8, "index": 58, "kind": "depth", "layer": 30, "offset": 2, "sources": 9, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 8, "index": 59, "kind": "depth", "layer": 30, "offset": 2, "sources": 9, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 8, "index": 60, "kind": "depth", "layer": 31, "offset": 3, "sources": 9, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 8, "index": 61, "kind": "depth", "layer": 31, "offset": 3, "sources": 9, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 8, "index": 62, "kind": "depth", "layer": 32, "offset": 4, "sources": 9, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 8, "index": 63, "kind": "depth", "layer": 32, "offset": 4, "sources": 9, "use_uniform": false }, { "branch": "output", "coefficient": "detached_learned", "group": null, "index": 64, "kind": "output", "layer": null, "offset": null, "sources": 9, "use_uniform": false } ], "visits_sha256": "185590e26f481634a0c9f936aa00f6f1b54ddf67e90abdc33fea58763842862c" } }, "uniform_group_6_only": { "bits_per_byte": 7.959341947457508, "forward": { "activation_sha256": { "attention_branch_output": "676759a2d7c0615ad7e1f246e90927fb829d00c8b6c3745032ae6b27ab7d902a", "mlp_branch_output": "afe29f3ef4d2992353f0bb59fe7c5c2c5050fca671a53184825a8d63cd83a2bc", "post_attention_state": "31853ffb07b24cc59e774bb20656840b4358b3080b42516d380f2dc0f46404c8", "post_mlp_state": "7fd9c911e93175896ad618842c120e77b3f017c17b7eec9c2bfb01fc6dbc8ac2", "pre_attention_input": "de71d4cf2d509a1063f6eac7f50c4885e9a058dee7e4b993ad40f5527f528d00", "pre_mlp_input": "2999a257265024350fd2665e2a8d452a03cd662193b4245d15290408ad29922d" }, "logits_sha256": "873a309a197bfd08fa523edcb440d2697903a34abc02dad1c7f5991509c3c151", "loss_nats": 5.516995429992676, "mixer_summary_sha256": "8450b24bdc777885a3357dc7b8080e7410acbe96943686433da76f379295133a" }, "loss_nats": 5.516995429992676, "loss_scale": 1.0, "mixers": [ { "branch": "attention", "entropy_mean": -0.0, "group": 1, "latest_source_mass": 1.0, "layer": 1, "max_source_mass": 1.0, "mixer_index": 0, "normalized_entropy": 1.0, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 1.0, "median": 1.0, "p05": 1.0, "p95": 1.0 } ], "sources": 1 }, { "branch": "mlp", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 1, "max_source_mass": 0.5, "mixer_index": 1, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 2, "max_source_mass": 0.5, "mixer_index": 2, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 2, "max_source_mass": 0.5, "mixer_index": 3, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 3, "max_source_mass": 0.5, "mixer_index": 4, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 3, "max_source_mass": 0.5, "mixer_index": 5, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 4, "max_source_mass": 0.5, "mixer_index": 6, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 4, "max_source_mass": 0.5, "mixer_index": 7, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6931471824645996, "group": 2, "latest_source_mass": 0.5, "layer": 5, "max_source_mass": 0.5, "mixer_index": 8, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "completed_group_1", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 5, "max_source_mass": 0.3333333432674408, "mixer_index": 9, "normalized_entropy": 1.0000000180537645, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 6, "max_source_mass": 0.3333333432674408, "mixer_index": 10, "normalized_entropy": 1.0000000180537645, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 6, "max_source_mass": 0.3333333432674408, "mixer_index": 11, "normalized_entropy": 1.0000000180537645, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 7, "max_source_mass": 0.3333333432674408, "mixer_index": 12, "normalized_entropy": 1.0000000180537645, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 7, "max_source_mass": 0.3333333432674408, "mixer_index": 13, "normalized_entropy": 1.0000000180537645, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 8, "max_source_mass": 0.3333333432674408, "mixer_index": 14, "normalized_entropy": 1.0000000180537645, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 8, "max_source_mass": 0.3333333432674408, "mixer_index": 15, "normalized_entropy": 1.0000000180537645, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0986123085021973, "group": 3, "latest_source_mass": 0.3333333432674408, "layer": 9, "max_source_mass": 0.3333333432674408, "mixer_index": 16, "normalized_entropy": 1.0000000180537645, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_2", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 9, "max_source_mass": 0.25, "mixer_index": 17, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 10, "max_source_mass": 0.25, "mixer_index": 18, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 10, "max_source_mass": 0.25, "mixer_index": 19, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 11, "max_source_mass": 0.25, "mixer_index": 20, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 11, "max_source_mass": 0.25, "mixer_index": 21, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 12, "max_source_mass": 0.25, "mixer_index": 22, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 12, "max_source_mass": 0.25, "mixer_index": 23, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3862943649291992, "group": 4, "latest_source_mass": 0.25, "layer": 13, "max_source_mass": 0.25, "mixer_index": 24, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_3", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 13, "max_source_mass": 0.20000000298023224, "mixer_index": 25, "normalized_entropy": 1.0000000186840277, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 14, "max_source_mass": 0.20000000298023224, "mixer_index": 26, "normalized_entropy": 1.0000000186840277, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 14, "max_source_mass": 0.20000000298023224, "mixer_index": 27, "normalized_entropy": 1.0000000186840277, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 15, "max_source_mass": 0.20000000298023224, "mixer_index": 28, "normalized_entropy": 1.0000000186840277, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 15, "max_source_mass": 0.20000000298023224, "mixer_index": 29, "normalized_entropy": 1.0000000186840277, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 16, "max_source_mass": 0.20000000298023224, "mixer_index": 30, "normalized_entropy": 1.0000000186840277, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 16, "max_source_mass": 0.20000000298023224, "mixer_index": 31, "normalized_entropy": 1.0000000186840277, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.6094379425048828, "group": 5, "latest_source_mass": 0.20000000298023224, "layer": 17, "max_source_mass": 0.20000000298023224, "mixer_index": 32, "normalized_entropy": 1.0000000186840277, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_4", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 17, "max_source_mass": 0.1666666716337204, "mixer_index": 33, "normalized_entropy": 1.0000001451965654, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 18, "max_source_mass": 0.1666666716337204, "mixer_index": 34, "normalized_entropy": 1.0000001451965654, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 18, "max_source_mass": 0.1666666716337204, "mixer_index": 35, "normalized_entropy": 1.0000001451965654, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 19, "max_source_mass": 0.1666666716337204, "mixer_index": 36, "normalized_entropy": 1.0000001451965654, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 19, "max_source_mass": 0.1666666716337204, "mixer_index": 37, "normalized_entropy": 1.0000001451965654, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 20, "max_source_mass": 0.1666666716337204, "mixer_index": 38, "normalized_entropy": 1.0000001451965654, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 20, "max_source_mass": 0.1666666716337204, "mixer_index": 39, "normalized_entropy": 1.0000001451965654, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.791759729385376, "group": 6, "latest_source_mass": 0.1666666716337204, "layer": 21, "max_source_mass": 0.1666666716337204, "mixer_index": 40, "normalized_entropy": 1.0000001451965654, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_5", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 21, "max_source_mass": 0.1428571492433548, "mixer_index": 41, "normalized_entropy": 1.00000003408302, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 22, "max_source_mass": 0.1428571492433548, "mixer_index": 42, "normalized_entropy": 1.00000003408302, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 22, "max_source_mass": 0.1428571492433548, "mixer_index": 43, "normalized_entropy": 1.00000003408302, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 23, "max_source_mass": 0.1428571492433548, "mixer_index": 44, "normalized_entropy": 1.00000003408302, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 23, "max_source_mass": 0.1428571492433548, "mixer_index": 45, "normalized_entropy": 1.00000003408302, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 24, "max_source_mass": 0.1428571492433548, "mixer_index": 46, "normalized_entropy": 1.00000003408302, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 24, "max_source_mass": 0.1428571492433548, "mixer_index": 47, "normalized_entropy": 1.00000003408302, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9459102153778076, "group": 7, "latest_source_mass": 0.1428571492433548, "layer": 25, "max_source_mass": 0.1428571492433548, "mixer_index": 48, "normalized_entropy": 1.00000003408302, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_6", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 25, "max_source_mass": 0.125, "mixer_index": 49, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 26, "max_source_mass": 0.125, "mixer_index": 50, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 26, "max_source_mass": 0.125, "mixer_index": 51, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 27, "max_source_mass": 0.125, "mixer_index": 52, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 27, "max_source_mass": 0.125, "mixer_index": 53, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 28, "max_source_mass": 0.125, "mixer_index": 54, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 28, "max_source_mass": 0.125, "mixer_index": 55, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.079441547393799, "group": 8, "latest_source_mass": 0.125, "layer": 29, "max_source_mass": 0.125, "mixer_index": 56, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_7", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 29, "max_source_mass": 0.1111111119389534, "mixer_index": 57, "normalized_entropy": 1.0000000180537645, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 30, "max_source_mass": 0.1111111119389534, "mixer_index": 58, "normalized_entropy": 1.0000000180537645, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 30, "max_source_mass": 0.1111111119389534, "mixer_index": 59, "normalized_entropy": 1.0000000180537645, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 31, "max_source_mass": 0.1111111119389534, "mixer_index": 60, "normalized_entropy": 1.0000000180537645, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 31, "max_source_mass": 0.1111111119389534, "mixer_index": 61, "normalized_entropy": 1.0000000180537645, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 32, "max_source_mass": 0.1111111119389534, "mixer_index": 62, "normalized_entropy": 1.0000000180537645, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 32, "max_source_mass": 0.1111111119389534, "mixer_index": 63, "normalized_entropy": 1.0000000180537645, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "output", "entropy_mean": 2.1972246170043945, "group": null, "latest_source_mass": 0.1111111119389534, "layer": null, "max_source_mass": 0.1111111119389534, "mixer_index": null, "normalized_entropy": 1.0000000180537645, "offset": null, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_8", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 } ], "mode": "uniform_group_6_only", "optimizer_state_after": "b82a349700eda58e2330a22ca2ee1f7fb1406efd47fd9aa306122b368a114192", "optimizer_state_before": "b82a349700eda58e2330a22ca2ee1f7fb1406efd47fd9aa306122b368a114192", "optimizer_state_unchanged": true, "positions": { "attention_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "676759a2d7c0615ad7e1f246e90927fb829d00c8b6c3745032ae6b27ab7d902a", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.0022265110164880753, 0.0021613112185150385, 0.0025601957459002733, 0.0025323182344436646, 0.0027253651060163975, 0.002904455177485943, 0.0029208383057266474, 0.00299624796025455, 0.0030937574338167906, 0.0033804215490818024, 0.0034370317589491606, 0.0038969130255281925, 0.004255065228790045, 0.004119740333408117, 0.003970506135374308, 0.004385598469525576, 0.0041999719105660915, 0.005232699681073427, 0.004664790350943804, 0.004898969549685717, 0.005131533369421959, 0.005069608334451914, 0.0051318020559847355, 0.0058372304774820805, 0.0059690922498703, 0.00554921617731452, 0.0058550527319312096, 0.006255472078919411, 0.006502739619463682, 0.006122888531535864, 0.007071305997669697, 0.006241494789719582 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.00015962477027642308, "first_to_last_ratio": 3.7699040072151497, "imbalance_abs_log_ratio": 1.3270495388586014, "last_quartile_mean": 4.234186599205714e-05, "mean": 8.617327489446325e-05, "normalized": [ 2.5428430303860874, 2.3222064178412967, 2.1011513502014654, 1.8911470095267005, 1.7004447966680725, 1.5353700031893112, 1.4229335846259188, 1.3028620492588128, 1.2049362956920768, 1.1705197150696323, 1.0773612761873523, 1.0133200843661747, 0.9906112963967714, 0.9272530070793641, 0.8550207003106389, 0.8185600560490035, 0.7703816559177782, 0.7173918665339755, 0.6760157258502337, 0.6514535263006792, 0.6184138115448249, 0.6053606047045522, 0.5838821473278404, 0.5697017460729616, 0.5457761369570343, 0.5276086337508958, 0.5104655265174319, 0.4951752820428687, 0.47527333481689216, 0.465789239220051, 0.45896414036514044, 0.45180594922816086 ], "peak_layer": 1, "peak_normalized": 2.5428430303860874, "population_cv": 0.5705341151299013, "rest_mean": 9.280180475562466e-05, "spike_contrast": 0.5428688997682254, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.0379213644191625e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00021912511147093028, 0.00020011213200632483, 0.00018106309289578348, 0.00016296633111778647, 0.00014653289690613747, 0.00013230786134954542, 0.0001226188469445333, 0.00011227188952034339, 0.0001038333066389896, 0.00010086751717608422, 9.283974941354245e-05, 8.732111018616706e-05, 8.536421955795959e-05, 7.990442827576771e-05, 7.367993384832516e-05, 7.053800072753802e-05, 6.63863102090545e-05, 6.182000652188435e-05, 5.825448897667229e-05, 5.6137883802875876e-05, 5.329074338078499e-05, 5.216590579948388e-05, 5.031503678765148e-05, 4.909306517220102e-05, 4.703131708083674e-05, 4.5465763832908124e-05, 4.3988486140733585e-05, 4.2670875700423494e-05, 4.095585973118432e-05, 4.013858415419236e-05, 3.955044303438626e-05, 3.893359826179221e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.00021442661636683624, "first_to_last_ratio": 1.843144421770586, "imbalance_abs_log_ratio": 0.6114730379455638, "last_quartile_mean": 0.0001163373926829081, "mean": 0.00015221274247778638, "normalized": [ 1.7741446897870732, 1.648461275666186, 1.5301464224684427, 1.4217121808823485, 1.3265065496562456, 1.2454420544341354, 1.1897998831173078, 1.1336248235930768, 1.0869089599614004, 1.0643003125599004, 1.0227030055526172, 0.9941765648180964, 0.9788958472869692, 0.9511102301475627, 0.9228060651723192, 0.9052418824754537, 0.8846484396889256, 0.865292475366091, 0.8497311645646503, 0.8388691935254817, 0.8256303537004397, 0.8176325372729673, 0.8076852907347736, 0.8000670639426607, 0.7892588692968658, 0.7814415503461742, 0.7731202151361662, 0.7664994567889658, 0.758490072461763, 0.7529625284402992, 0.7485174912671357, 0.7441725498875049 ], "peak_layer": 1, "peak_normalized": 1.7741446897870732, "population_cv": 0.27364872432853743, "rest_mean": 0.00015762320577373935, "spike_contrast": 0.7803181014868814, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012299624067964033, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00027004742878489196, 0.0002509168116375804, 0.0002329077833564952, 0.00021640271006617695, 0.00020191119983792305, 0.00018957215070258826, 0.0001811027032090351, 0.00017255214333999902, 0.00016544139361940324, 0.00016200006939470768, 0.00015566842921543866, 0.0001513263414381072, 0.00014900042151566595, 0.0001447710965294391, 0.00014046284195501357, 0.0001377893495373428, 0.00013465476513374597, 0.00013170854072086513, 0.00012933991092722863, 0.0001276865805266425, 0.00012567146040964872, 0.00012445409083738923, 0.00012293999316170812, 0.00012178040196886286, 0.00012013525702059269, 0.00011894536146428436, 0.00011767874821089208, 0.00011667098442558199, 0.00011545185407157987, 0.00011461049143690616, 0.00011393390013836324, 0.00011327254469506443 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.19015566632151604, "first_to_last_ratio": 1.8431444167877922, "imbalance_abs_log_ratio": 0.6114730352421437, "last_quartile_mean": 0.10316916275769472, "mean": 0.13498378219082952, "normalized": [ 1.7741447141792424, 1.6484611693507762, 1.5301464164964773, 1.421712099711509, 1.3265065258282553, 1.245442086656075, 1.1897999853143832, 1.1336247998774232, 1.0869090145314644, 1.0643003539468234, 1.0227030148964142, 0.9941765578539046, 0.978895844117059, 0.9511102293531465, 0.9228060470726628, 0.9052419251832852, 0.8846484748200404, 0.8652925213936268, 0.8497311905993837, 0.8388692019864461, 0.8256303029992552, 0.8176324960789627, 0.8076852727084025, 0.800067049482784, 0.789258877083647, 0.7814415614650243, 0.7731201949085273, 0.7664994759042474, 0.758490077799444, 0.7529625178867759, 0.7485174642151933, 0.7441725362993381 ], "peak_layer": 1, "peak_normalized": 1.7741447141792424, "population_cv": 0.2736487184871982, "rest_mean": 0.13978183573042904, "spike_contrast": 0.7803180757143806, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.10907429307699204, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.23948076367378235, 0.2225155234336853, 0.20654495060443878, 0.1919080764055252, 0.17905686795711517, 0.16811448335647583, 0.16060370206832886, 0.1530209630727768, 0.14671508967876434, 0.14366328716278076, 0.13804832100868225, 0.13419771194458008, 0.1321350634098053, 0.12838445603847504, 0.12456385046243668, 0.12219297885894775, 0.11941319704055786, 0.116800457239151, 0.11469992995262146, 0.11323373764753342, 0.11144670099020004, 0.11036712676286697, 0.10902441293001175, 0.10799607634544373, 0.10653714835643768, 0.10548193752765656, 0.10435868799686432, 0.1034649983048439, 0.10238385945558548, 0.10163772851228714, 0.10103771835565567, 0.10045122355222702 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 6.883365585963475e-05, "first_to_last_ratio": 2.2497546817252747, "imbalance_abs_log_ratio": 0.8108211799277664, "last_quartile_mean": 3.059607183786284e-05, "mean": 4.691841348858361e-05, "normalized": [ 1.7359303646705315, 1.6475859007944922, 1.5536416135247637, 1.4921074478374423, 1.436565604934822, 1.343834173349448, 1.2787439414688093, 1.2483315192759759, 1.191227587726315, 1.1603086071531024, 1.0995306735006558, 1.059977946939621, 1.0583984121521266, 1.0176934649371145, 0.9754909410255406, 0.9631621800279858, 0.9242301483750994, 0.8752327792179452, 0.8454160845678106, 0.8232909667825862, 0.7863453139125789, 0.77746685672799, 0.7497386589361914, 0.7388508716017261, 0.7093180450413769, 0.6908354487677191, 0.6685471901846501, 0.6592354484095886, 0.631440334976849, 0.6235901145414114, 0.6199506557967245, 0.6139807028410055 ], "peak_layer": 1, "peak_normalized": 1.7359303646705315, "population_cv": 0.3220412433027962, "rest_mean": 4.907019663530971e-05, "spike_contrast": 0.7193528234378936, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.529878449626267e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 8.144709863699973e-05, 7.730211655143648e-05, 7.289439963642508e-05, 7.000731420703232e-05, 6.740137905580923e-05, 6.305056740529835e-05, 5.999663699185476e-05, 5.856973439222202e-05, 5.5890508519951254e-05, 5.4439839004771784e-05, 5.1588234782684594e-05, 4.9732483603293076e-05, 4.965837433701381e-05, 4.774856279254891e-05, 4.576848732540384e-05, 4.519004141911864e-05, 4.3363412260077894e-05, 4.106453343410976e-05, 3.966558142565191e-05, 3.862750600092113e-05, 3.6894074582960457e-05, 3.647751145763323e-05, 3.517654840834439e-05, 3.4665710700210184e-05, 3.328007733216509e-05, 3.241290323785506e-05, 3.136717350571416e-05, 3.0930281354812905e-05, 2.9626178729813546e-05, 2.925785884144716e-05, 2.9087101211189292e-05, 2.8807000489905477e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 0.0001734125216898974, "first_to_last_ratio": 1.495517223041696, "imbalance_abs_log_ratio": 0.4024721155990803, "last_quartile_mean": 0.00011595488103921525, "mean": 0.00013990532806928968, "normalized": [ 1.3768889945483427, 1.3340940553978193, 1.2910049686488183, 1.2527203273458163, 1.2147536525239127, 1.1769854337196524, 1.14744939519443, 1.1220955988146075, 1.0944022472959734, 1.0773022658134643, 1.0508559212107247, 1.03206209473486, 1.0208171898483585, 1.000561882500425, 0.9805161605170362, 0.9663129296065233, 0.9487424991891081, 0.931931984738623, 0.9181974345490227, 0.9075104531838204, 0.8943640941785878, 0.8862664021756573, 0.8757830774574916, 0.86790401991425, 0.8562894022841879, 0.8478155487572495, 0.8387227683262297, 0.8314011142413483, 0.8224071457687988, 0.8161875050523302, 0.8112199678685613, 0.8064334645939688 ], "peak_layer": 1, "peak_normalized": 1.3768889945483427, "population_cv": 0.16325957486062354, "rest_mean": 0.0001431148236776978, "spike_contrast": 0.8564734849544942, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001225740517838858, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0001926341064972803, 0.00018664686649572104, 0.00018061847367789596, 0.00017526224837638438, 0.0001699505082797259, 0.00016466653323732316, 0.00016053428407758474, 0.00015698715287726372, 0.00015311270544771105, 0.00015072032692842185, 0.00014702034241054207, 0.00014439098595175892, 0.00014281776384450495, 0.00013998393842484802, 0.00013717943511437625, 0.00013519232743419707, 0.00013273413060232997, 0.00013038225006312132, 0.00012846071331296116, 0.00012696554767899215, 0.0001251263020094484, 0.00012399339175317436, 0.0001225267187692225, 0.00012142439663875848, 0.00011979944974882528, 0.00011861391249112785, 0.00011734178406186402, 0.00011631744564510882, 0.00011505914153531194, 0.00011418898066040128, 0.00011349399574100971, 0.00011282433843007311 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 0.00014675117381557357, "first_to_last_ratio": 1.2676837465168667, "imbalance_abs_log_ratio": 0.23719141364257146, "last_quartile_mean": 0.00011576323686313117, "mean": 0.00013042020850662084, "normalized": [ 1.169988069421413, 1.156175152925158, 1.1417665260225645, 1.1306726331060184, 1.1183621029407895, 1.1062694829504214, 1.0940115056230622, 1.0844989956600726, 1.0711591662817939, 1.0579765491085218, 1.0443900226505625, 1.035471661013805, 1.0260498632741444, 1.013992390086173, 1.0062479316494695, 0.9977001177294099, 0.9863163473759529, 0.9769959732660659, 0.9687585669786032, 0.9602628595878113, 0.9503845997255937, 0.9426104617668664, 0.9330243108845692, 0.9259760938667744, 0.914797625470175, 0.906289421438531, 0.8956357003805738, 0.888470885240542, 0.8806216558595509, 0.8756991509238615, 0.8712858841028995, 0.8681382926882502 ], "peak_layer": 1, "peak_normalized": 1.169988069421413, "population_cv": 0.09123006364857297, "rest_mean": 0.00013202972385678786, "spike_contrast": 0.9219804606102014, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012172882561571896, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00015259008796419948, 0.00015078860451467335, 0.00014890942838974297, 0.0001474625605624169, 0.00014585701865144074, 0.00014427989663090557, 0.00014268120867200196, 0.00014144058513920754, 0.00013970080181024969, 0.0001379815221298486, 0.00013620956451632082, 0.00013504642993211746, 0.0001338176371064037, 0.00013224509893916547, 0.00013123506505507976, 0.0001301202573813498, 0.00012863558367826045, 0.00012742001854348928, 0.00012634569429792464, 0.00012523768236860633, 0.00012394935765769333, 0.00012293545296415687, 0.00012168522516731173, 0.00012076599523425102, 0.00011930809705518186, 0.00011819845531135798, 0.00011680899478960782, 0.00011587455810513347, 0.00011485085997264832, 0.00011420886585256085, 0.00011363328667357564, 0.00011322277714498341 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00030002388848515693, "first_to_last_ratio": 2.2836575149168885, "imbalance_abs_log_ratio": 0.8257783308640597, "last_quartile_mean": 0.0001313786706305109, "mean": 0.0001947244400071213, "normalized": [ 1.8998833386658438, 1.7835645331795598, 1.6713073472446418, 1.5703972875779748, 1.4780250735507643, 1.3761669788219328, 1.3066447563403325, 1.2401013786965793, 1.1762106459027857, 1.150547637187856, 1.0768735950666006, 1.0264900078881174, 1.0140784159549234, 0.9666871251601796, 0.9096841069789114, 0.8754807117769676, 0.8421876125778792, 0.8073809902734768, 0.77911667677541, 0.7607651063686129, 0.7403822030704592, 0.7308855611321243, 0.7152791523187013, 0.7043381880055872, 0.6944157299089508, 0.686864629891504, 0.680599049276497, 0.6764772714226681, 0.6702412094778135, 0.6672258960044359, 0.6635863560708457, 0.6581114274310631 ], "peak_layer": 1, "peak_normalized": 1.8998833386658438, "population_cv": 0.3592853832280775, "rest_mean": 0.00020492727344390005, "spike_contrast": 0.6813594750078023, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013962913944851608, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0003699537192005664, 0.0003473036049399525, 0.0003254443872720003, 0.00030579473241232336, 0.00028780760476365685, 0.00026797334430739284, 0.0002544356684666127, 0.0002414780465187505, 0.0002290369593538344, 0.00022403974435292184, 0.00020969360775779933, 0.00019988269195891917, 0.0001974658516701311, 0.00018823760910890996, 0.00017713772831484675, 0.00017047749133780599, 0.00016399451124016196, 0.0001572168112033978, 0.00015171305858530104, 0.00014813955931458622, 0.00014417050988413393, 0.00014232128160074353, 0.00013928233238402754, 0.0001371518592350185, 0.00013521971413865685, 0.00013374933041632175, 0.00013252926873974502, 0.00013172665785532445, 0.00013051234418526292, 0.00012992518895771354, 0.00012921648158226162, 0.000128150379168801 ] } } }, "mlp_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "afe29f3ef4d2992353f0bb59fe7c5c2c5050fca671a53184825a8d63cd83a2bc", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.0027073738165199757, 0.002722575329244137, 0.0026850115973502398, 0.0026861224323511124, 0.00271838391199708, 0.0026645169127732515, 0.0026859452482312918, 0.0027359756641089916, 0.0026767742820084095, 0.0027451682835817337, 0.0026999011170119047, 0.002663193503394723, 0.002626648638397455, 0.0026531771291047335, 0.0026517712976783514, 0.002676903735846281, 0.002670086920261383, 0.0026355870068073273, 0.0026610493659973145, 0.0026388936676084995, 0.0026469926815479994, 0.00264862016774714, 0.002641494618728757, 0.002674198243767023, 0.002557571977376938, 0.00252733426168561, 0.0025468147359788418, 0.0026856842450797558, 0.002551089273765683, 0.0025778315030038357, 0.0026001501828432083, 0.0025662274565547705 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.00015858857113926206, "first_to_last_ratio": 3.757867690304466, "imbalance_abs_log_ratio": 1.3238516928952826, "last_quartile_mean": 4.220174423608114e-05, "mean": 8.573221236929385e-05, "normalized": [ 2.5461641896458365, 2.3032502265274117, 2.0924715218207024, 1.8886722264368014, 1.7031663195050915, 1.537434754996766, 1.4223955235407997, 1.3049501874408618, 1.2055297161045848, 1.1692542365736316, 1.0769571962730058, 1.0144505847411098, 0.9913601784485884, 0.9317001386730132, 0.857757082878185, 0.8155634736467603, 0.7682612945963089, 0.7217806418748633, 0.678338533281248, 0.6511820384364709, 0.6192882306865346, 0.6053793146733001, 0.5848721334223365, 0.5718144052517429, 0.5462281883927541, 0.527515083157495, 0.5112993585545178, 0.496920907994256, 0.47722358366421574, 0.46552107689700334, 0.4601234027970099, 0.45317424906679377 ], "peak_layer": 1, "peak_normalized": 2.5461641896458365, "population_cv": 0.5686091737198813, "rest_mean": 9.231269224848668e-05, "spike_contrast": 0.5437781067692293, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.0197621021652594e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00021828828903380781, 0.00019746273756027222, 0.00017939221288543195, 0.0001619200484128669, 0.00014601621660403907, 0.000131807682919316, 0.00012194511509733275, 0.00011187626660102978, 0.00010335272963857278, 0.00010024275252362713, 9.232992306351662e-05, 8.697109296917915e-05, 8.499150135321543e-05, 7.987671415321529e-05, 7.353741239057854e-05, 6.992006092332304e-05, 6.586474046343938e-05, 6.187985127326101e-05, 5.8155463193543255e-05, 5.582727681030519e-05, 5.309295011102222e-05, 5.1900507969548926e-05, 5.0142381951445714e-05, 4.902291402686387e-05, 4.682935104938224e-05, 4.522503513726406e-05, 4.3834825191879645e-05, 4.260212881490588e-05, 4.0913433622336015e-05, 3.991015182691626e-05, 3.944739728467539e-05, 3.885163096128963e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.00021242323236947414, "first_to_last_ratio": 1.8345651540958352, "imbalance_abs_log_ratio": 0.6068074801791611, "last_quartile_mean": 0.0001157894184871111, "mean": 0.00015111673860701558, "normalized": [ 1.771707915810251, 1.6366668487510905, 1.5242909365839221, 1.4186705636795862, 1.3264449439106374, 1.2453857912414694, 1.1893415085703798, 1.1330085309323608, 1.0858463851390703, 1.062861239127963, 1.023237806768965, 0.9942126959777651, 0.978436257513136, 0.9531027445378829, 0.9232414968561398, 0.904013237249231, 0.8853704564278231, 0.8680164030054869, 0.8515402788738432, 0.8395817783841374, 0.8267435189825344, 0.8185143642831039, 0.809172703595369, 0.8007918347797284, 0.7909896391756025, 0.7821052393452608, 0.7753388667989797, 0.7682453290925442, 0.7604995795745633, 0.7549901567959042, 0.7508572350862155, 0.7467737131490535 ], "peak_layer": 1, "peak_normalized": 1.771707915810251, "population_cv": 0.2714170004285523, "rest_mean": 0.00015645500981783562, "spike_contrast": 0.7816309251520459, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012229007406858728, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0002677347220014781, 0.00024732775636948645, 0.00023034587502479553, 0.00021438486874103546, 0.00020044803386554122, 0.0001881986390799284, 0.00017972940986510366, 0.00017121655400842428, 0.00016408956435043365, 0.00016061612404882908, 0.0001546283601783216, 0.00015024218009784818, 0.00014785809617023915, 0.0001440297783119604, 0.00013951724395155907, 0.000136611532070674, 0.0001337942958343774, 0.00013117180787958205, 0.0001286819897359237, 0.00012687486014328897, 0.00012493478425312787, 0.00012369122123345733, 0.00012227953993715346, 0.00012101305037504062, 0.00011953177454415709, 0.00011818919301731512, 0.00011716668086592108, 0.00011609472858253866, 0.00011492421617731452, 0.00011409165017539635, 0.00011346709652571008, 0.00011285000800853595 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.18837904930114746, "first_to_last_ratio": 1.8345651581276443, "imbalance_abs_log_ratio": 0.6068074823768531, "last_quartile_mean": 0.10268321540206671, "mean": 0.13401183625683188, "normalized": [ 1.7717078885231814, 1.6366668119471057, 1.524290957183596, 1.418670514273678, 1.326444920903592, 1.2453858718827995, 1.1893415500975306, 1.1330085716119305, 1.0858463365841577, 1.062861212873729, 1.0232377434953237, 0.9942127340740732, 0.9784362443707608, 0.9531027271661117, 0.9232414917376033, 0.904013241653558, 0.885370477646398, 0.8680164390463408, 0.8515402681847718, 0.8395816969955481, 0.8267435345782925, 0.8185144279720658, 0.8091727254708025, 0.8007918405918669, 0.7909896328038464, 0.7821052665924377, 0.7753388466443417, 0.7682453528580236, 0.7604996016741158, 0.7549901615676585, 0.7508572336421532, 0.7467736753526057 ], "peak_layer": 1, "peak_normalized": 1.7717078885231814, "population_cv": 0.27141699472708714, "rest_mean": 0.1387458680956452, "spike_contrast": 0.7816309473985974, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.10844806432724, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.23742982745170593, 0.2193327248096466, 0.2042730301618576, 0.19011864066123962, 0.1777593195438385, 0.16689644753932953, 0.15938584506511688, 0.151836559176445, 0.1455162614583969, 0.1424359828233719, 0.13712596893310547, 0.13323627412319183, 0.13112203776836395, 0.12772704660892487, 0.1237252876162529, 0.1211484745144844, 0.11865012347698212, 0.11632447689771652, 0.11411647498607635, 0.11251388490200043, 0.11079341918230057, 0.10969062149524689, 0.10843872278928757, 0.10731558501720428, 0.10600197315216064, 0.10481136292219162, 0.10390458256006241, 0.1029539704322815, 0.10191594809293747, 0.10117761790752411, 0.10062375664710999, 0.10007651150226593 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 6.873187476230669e-05, "first_to_last_ratio": 2.248757500145603, "imbalance_abs_log_ratio": 0.8103778415280068, "last_quartile_mean": 3.0564378221242805e-05, "mean": 4.685761763312257e-05, "normalized": [ 1.7400006587426948, 1.6405194725375354, 1.5551306973978087, 1.4944078341288562, 1.4327724019957897, 1.3414187773842097, 1.2804088050749736, 1.2499327782691996, 1.1927955171655855, 1.1608931065609767, 1.0981756972550076, 1.0577423904261343, 1.0542786038595535, 1.0218640087393278, 0.9775807389332813, 0.9570972402588515, 0.9209982861589985, 0.8815092249082567, 0.8478232134430647, 0.8204952153707635, 0.7871780668180614, 0.7759097745574741, 0.7506963524238334, 0.7421155343193898, 0.7072927325438048, 0.6891814936654451, 0.6695399899996352, 0.6616343983211914, 0.6345111343049281, 0.6212687927046295, 0.6200231913288072, 0.6148038704019305 ], "peak_layer": 1, "peak_normalized": 1.7400006587426948, "population_cv": 0.3217536161653951, "rest_mean": 4.900405669845727e-05, "spike_contrast": 0.719671983430414, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.526684668031521e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 8.153228554874659e-05, 7.687083416385576e-05, 7.286971958819777e-05, 7.002439087955281e-05, 6.71363013680093e-05, 6.285568815656006e-05, 5.999690620228648e-05, 5.8568872191244736e-05, 5.5891556257847697e-05, 5.439668530016206e-05, 5.145789691596292e-05, 4.9563288484932855e-05, 4.9400983698433265e-05, 4.7882112994557247e-05, 4.580710447044112e-05, 4.484729652176611e-05, 4.3155785533599555e-05, 4.130542220082134e-05, 3.972697595600039e-05, 3.844645107164979e-05, 3.688528886414133e-05, 3.635728353401646e-05, 3.517584264045581e-05, 3.477376594673842e-05, 3.314205241622403e-05, 3.229340290999971e-05, 3.1373048841487616e-05, 3.10026116494555e-05, 2.9731680115219206e-05, 2.9111175535945222e-05, 2.9052809622953646e-05, 2.8808244678657502e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 0.0001719075116852764, "first_to_last_ratio": 1.4894635014227988, "imbalance_abs_log_ratio": 0.39841598895878805, "last_quartile_mean": 0.00011541572621354135, "mean": 0.00013894635094402474, "normalized": [ 1.3738985779819257, 1.3296082759737424, 1.2883384004275333, 1.2499940655466777, 1.2130730176151365, 1.1755503979222366, 1.147409009170091, 1.11990595257071, 1.0929037170303153, 1.075598370036652, 1.0512762916941902, 1.0312594730951, 1.0194418974669357, 1.0018996501730997, 0.9803460168062212, 0.9648004557293595, 0.9493010807852901, 0.9344788929945884, 0.9198953859218769, 0.908001147597105, 0.8952913731815082, 0.886953676542665, 0.8771894463102521, 0.8683888934041172, 0.8579399877904575, 0.8482799643655919, 0.8408636339472488, 0.8330870307470335, 0.82434015220157, 0.8181187963300758, 0.8135579418994486, 0.809009026741244 ], "peak_layer": 1, "peak_normalized": 1.3738985779819257, "population_cv": 0.16161423472023242, "rest_mean": 0.0001421073115866252, "spike_contrast": 0.8576417505420824, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012187716347398237, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0001908981939777732, 0.0001847442181315273, 0.00017900991952046752, 0.00017368211410939693, 0.00016855206922627985, 0.00016333843814209104, 0.00015942829486448318, 0.00015560684551019222, 0.0001518549834145233, 0.00014945046859793365, 0.00014607100456487387, 0.0001432897406630218, 0.00014164773165248334, 0.00013921030040364712, 0.000136215501697734, 0.0001340555027127266, 0.0001319019211223349, 0.00012984243221580982, 0.0001278161071240902, 0.00012616344611160457, 0.00012439746933523566, 0.00012323897681199014, 0.00012188227265141904, 0.00012065946793882176, 0.00011920763063244522, 0.00011786540562752634, 0.00011683493357850239, 0.00011575440294109285, 0.0001145390560850501, 0.00011367462138878182, 0.00011304090730845928, 0.00011240885214647278 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 0.00014552221000485588, "first_to_last_ratio": 1.262697055715921, "imbalance_abs_log_ratio": 0.2332499537229619, "last_quartile_mean": 0.00011524712863320019, "mean": 0.00012953460168318998, "normalized": [ 1.1676213631634658, 1.1525055688027706, 1.1391580058081985, 1.1289307982409096, 1.1166731577821778, 1.1054001772932092, 1.094813144753513, 1.0822854389705017, 1.0679308603491235, 1.0564894822224282, 1.0456838360716472, 1.0337072702557388, 1.0234945545465333, 1.0158819477934218, 1.0051772952892828, 0.9951467950359763, 0.9866643397779161, 0.9787559418439278, 0.970448736962688, 0.9607908684400951, 0.9509939230158972, 0.9428061355388626, 0.934924980051271, 0.9261034265480621, 0.9170582603044295, 0.9058562786860701, 0.8981778407016658, 0.8907564927732574, 0.8831168120837269, 0.8778216107141017, 0.8745192645108896, 0.8703053916682415 ], "peak_layer": 1, "peak_normalized": 1.1676213631634658, "population_cv": 0.08967236486753827, "rest_mean": 0.00013110875065817877, "spike_contrast": 0.9231588022206524, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012103419721825049, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00015124736819416285, 0.00014928934979252517, 0.000147560378536582, 0.00014623560127802193, 0.00014464781270362437, 0.00014318757166620344, 0.00014181618462316692, 0.00014019341324456036, 0.0001383339986205101, 0.00013685194426216185, 0.00013545223919209093, 0.00013390085950959474, 0.00013257795944809914, 0.00013159186346456409, 0.00013020524056628346, 0.00012890594371128827, 0.00012780717224813998, 0.00012678276107180864, 0.0001257066905964166, 0.0001244556624442339, 0.00012318661902099848, 0.0001221260172314942, 0.00012110513489460573, 0.0001199624384753406, 0.00011879077646881342, 0.00011733973224181682, 0.00011634510883595794, 0.00011538378748809919, 0.0001143941844929941, 0.00011370827269274741, 0.00011328050459269434, 0.0001127346622524783 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00029741139769612346, "first_to_last_ratio": 2.278072415841521, "imbalance_abs_log_ratio": 0.8233296537390045, "last_quartile_mean": 0.00013055396993877366, "mean": 0.00019328556300024502, "normalized": [ 1.889398576482247, 1.780233894901915, 1.6772594716902607, 1.566224958959574, 1.4762232610465842, 1.375571144634086, 1.3083686481730985, 1.2364401527267181, 1.173148897934763, 1.1508864184295666, 1.080457040307468, 1.0318582947152708, 1.0081957004611959, 0.9698994715426978, 0.9066127339785177, 0.8754550048258108, 0.8446146862114313, 0.8115724449181009, 0.7806677558079268, 0.7613480996239201, 0.7403134026279838, 0.7294052762839086, 0.7160186976339193, 0.7062575707090604, 0.6942777320753531, 0.6866005528973866, 0.6829938476675228, 0.6753403086482458, 0.6715967314167978, 0.6686860556999406, 0.6640286432895989, 0.6600445236791291 ], "peak_layer": 1, "peak_normalized": 1.889398576482247, "population_cv": 0.3579154260008552, "rest_mean": 0.00020340604748559632, "spike_contrast": 0.681567477924495, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013863494677934796, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0003651934675872326, 0.0003440935106482357, 0.00032419004128314555, 0.0003027286729775369, 0.0002853326441254467, 0.00026587804313749075, 0.00025288877077400684, 0.00023898603103589267, 0.00022675274522043765, 0.00022244972933549434, 0.0002088367473334074, 0.00019944331143051386, 0.00019486967357806861, 0.00018746756541077048, 0.00017523515271022916, 0.00016921281348913908, 0.0001632518251426518, 0.00015686523693148047, 0.00015089180669747293, 0.00014715759607497603, 0.00014309189282357693, 0.00014098350948188454, 0.00013839607709087431, 0.0001365093921776861, 0.0001341938623227179, 0.00013270997442305088, 0.00013201285037212074, 0.00013053353177383542, 0.0001298099523410201, 0.00012924736074637622, 0.00012834715016651899, 0.00012757707736454904 ] } } }, "post_attention_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "31853ffb07b24cc59e774bb20656840b4358b3080b42516d380f2dc0f46404c8", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.0022265110164880753, 0.0040087271481752396, 0.005379301495850086, 0.006410556845366955, 0.0027253651060163975, 0.004838948138058186, 0.006175771821290255, 0.00742313964292407, 0.0030937574338167906, 0.005150026176124811, 0.007019025273621082, 0.008143599145114422, 0.004255065228790045, 0.0066224560141563416, 0.008129785768687725, 0.010099578648805618, 0.0041999719105660915, 0.00719039561226964, 0.009030314162373543, 0.010736005380749702, 0.005131533369421959, 0.00787449348717928, 0.009912566281855106, 0.012113703414797783, 0.0059690922498703, 0.00876442901790142, 0.010806876234710217, 0.012353328987956047, 0.006502739619463682, 0.009470963850617409, 0.012123330496251583, 0.01384514570236206 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.00015962236011546338, "first_to_last_ratio": 3.7698422272252903, "imbalance_abs_log_ratio": 1.3270331510418396, "last_quartile_mean": 4.2341920561739244e-05, "mean": 8.61727822893954e-05, "normalized": [ 2.542734798705971, 2.3221448837042376, 2.101145461274622, 1.8911858524935854, 1.7004204056817789, 1.5354142426015858, 1.4229001770142031, 1.3028733810328659, 1.2049412416934622, 1.1705258152816296, 1.0773908232649334, 1.0133220774376726, 0.990609360100461, 0.9272534949358001, 0.8550385909364457, 0.8185651575045715, 0.7703972051466339, 0.7173947009730917, 0.6760256695667757, 0.6514523109028573, 0.618419077603279, 0.6053620388084041, 0.5838930841859182, 0.5697043695032278, 0.5457802700901755, 0.5276149005446632, 0.5104701332743111, 0.49517988582787786, 0.47527562953524777, 0.46579059316028215, 0.45896435764104826, 0.45181000957238127 ], "peak_layer": 1, "peak_normalized": 2.542734798705971, "population_cv": 0.5705202801130907, "rest_mean": 9.280119640472414e-05, "spike_contrast": 0.542873885449774, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.03793460666202e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00021911453222855926, 0.00020010568550787866, 0.00018106155039276928, 0.00016296874673571438, 0.00014652995741926134, 0.00013231091725174338, 0.0001226152671733871, 0.00011227222421439365, 0.00010383313929196447, 0.00010086746624438092, 9.284176485380158e-05, 8.732078276807442e-05, 8.536356472177431e-05, 7.99040135461837e-05, 7.368105434579775e-05, 7.053803710732609e-05, 6.638727063545957e-05, 6.181989738252014e-05, 5.825501284562051e-05, 5.613745815935545e-05, 5.329089253791608e-05, 5.216573117650114e-05, 5.0315691623836756e-05, 4.9093010602518916e-05, 4.703140439232811e-05, 4.546604395727627e-05, 4.398863165988587e-05, 4.267102849553339e-05, 4.095582335139625e-05, 4.013847137684934e-05, 3.955023566959426e-05, 3.893372559105046e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.00021442477373057045, "first_to_last_ratio": 1.8431254562763404, "imbalance_abs_log_ratio": 0.6114627481432037, "last_quartile_mean": 0.00011633759004325839, "mean": 0.00015221248349917005, "normalized": [ 1.7740805953138137, 1.6484457246979003, 1.5301434809496548, 1.4217241600875643, 1.3264958046529274, 1.2454577490392524, 1.1897861330408546, 1.1336265611690741, 1.086914920172498, 1.064301358569379, 1.0227209980571423, 0.9941753882589759, 0.9788979908211966, 0.9511107967643203, 0.9228143274453928, 0.9052447611165039, 0.8846530997418666, 0.8652930871753776, 0.8497381552743479, 0.8388656494639406, 0.8256334792987466, 0.8176309647342147, 0.8076951735882674, 0.8000661307359944, 0.7892605467734257, 0.78144613040264, 0.7731219607566042, 0.7664988488799842, 0.7584935618388989, 0.7529639051546407, 0.7485195296370439, 0.7441790263875565 ], "peak_layer": 1, "peak_normalized": 1.7740805953138137, "population_cv": 0.2736411578809574, "rest_mean": 0.00015762286892384981, "spike_contrast": 0.7803207938393821, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012299640220589936, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00027003721334040165, 0.00025091401766985655, 0.00023290693934541196, 0.0002164041652576998, 0.000201909220777452, 0.0001895742170745507, 0.00018110030214302242, 0.00017255211423616856, 0.00016544201935175806, 0.00016199995297938585, 0.00015567090304102749, 0.00015132590488065034, 0.0001490004942752421, 0.00014477093645837158, 0.00014046386058907956, 0.00013778955326415598, 0.0001346552453469485, 0.00013170840975362808, 0.00012934075493831187, 0.00012768582382705063, 0.00012567172234412283, 0.00012445363972801715, 0.00012294128828216344, 0.00012178005272289738, 0.00012013530795229599, 0.00011894585622940212, 0.00011767881369451061, 0.00011667069338727742, 0.00011545218876563013, 0.00011461050598882139, 0.00011393401655368507, 0.00011327333777444437 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.190154030919075, "first_to_last_ratio": 1.84312545379361, "imbalance_abs_log_ratio": 0.6114627467961817, "last_quartile_mean": 0.10316933691501617, "mean": 0.13498355145566165, "normalized": [ 1.774080628234443, 1.6484456620184311, 1.5301434020504485, 1.4217242444591272, 1.326495766996536, 1.2454576834375484, 1.189786122599316, 1.133626627255238, 1.0869149569673349, 1.0643014004698326, 1.0227209907477024, 0.9941753870542451, 0.9788979589698585, 0.9511107512164827, 0.9228143032197887, 0.9052447972745712, 0.8846531331847195, 0.8652931173486506, 0.8497382179112226, 0.8388656682549475, 0.8256334805754935, 0.8176309131099365, 0.8076951535479161, 0.8000661540388709, 0.7892605573866218, 0.7814461538045975, 0.7731219580171443, 0.7664988542586116, 0.7584935821787124, 0.7529638601652359, 0.7485195164462104, 0.7441789968002049 ], "peak_layer": 1, "peak_normalized": 1.774080628234443, "population_cv": 0.27364115345546325, "rest_mean": 0.13978153549962574, "spike_contrast": 0.7803207857775153, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.10907443761825561, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.2394717037677765, 0.22251304984092712, 0.2065441906452179, 0.19190938770771027, 0.1790551096200943, 0.16811630129814148, 0.16060155630111694, 0.1530209481716156, 0.14671564102172852, 0.1436631828546524, 0.13805051147937775, 0.13419732451438904, 0.13213512301445007, 0.1283843070268631, 0.1245647519826889, 0.12219315767288208, 0.11941362172365189, 0.11680033802986145, 0.11470068246126175, 0.1132330670952797, 0.11144693940877914, 0.11036672443151474, 0.10902556031942368, 0.10799577087163925, 0.10653719305992126, 0.10548237711191177, 0.1043587476015091, 0.10346473753452301, 0.10238415747880936, 0.10163773596286774, 0.10103782266378403, 0.10045192390680313 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 6.883367768750759e-05, "first_to_last_ratio": 2.249759357551583, "imbalance_abs_log_ratio": 0.810823258297238, "last_quartile_mean": 3.059601795030176e-05, "mean": 4.691846504556452e-05, "normalized": [ 1.7359059710084281, 1.6475789727938472, 1.5536456441204052, 1.4921194549590373, 1.4365657321885223, 1.3438513058553838, 1.2787343947807137, 1.2483299149152907, 1.1912354282513808, 1.1603083401306369, 1.099547144002948, 1.059972440022979, 1.0583955432739494, 1.0176800955770247, 0.9754963047746087, 0.9631595708780456, 0.9242400656751482, 0.8752301116137658, 0.8454245377074081, 0.8232853322610043, 0.7863494898171388, 0.7774642190168813, 0.7497486129019063, 0.7388483538626245, 0.709315327139865, 0.6908343794807076, 0.6685469983108738, 0.6592365073812129, 0.6314341746588822, 0.6235898169926835, 0.6199474545603838, 0.6139783610863121 ], "peak_layer": 1, "peak_normalized": 1.7359059710084281, "population_cv": 0.3220406003070692, "rest_mean": 4.9070239684725595e-05, "spike_contrast": 0.7193541792517962, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.52988819940947e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 8.144604362314567e-05, 7.730187644483522e-05, 7.28946688468568e-05, 7.000795449130237e-05, 6.740145909134299e-05, 6.305144052021205e-05, 5.999625500408001e-05, 5.8569723478285596e-05, 5.589093780145049e-05, 5.4439886298496276e-05, 5.158906424185261e-05, 4.973227987647988e-05, 4.9658294301480055e-05, 4.774798799189739e-05, 4.576878927764483e-05, 4.51899686595425e-05, 4.3363925215089694e-05, 4.1064453398576006e-05, 3.9666021621087566e-05, 3.86272840842139e-05, 3.689431105158292e-05, 3.6477427784120664e-05, 3.5177054087398574e-05, 3.466563066467643e-05, 3.3279986382694915e-05, 3.2412888685939834e-05, 3.136719897156581e-05, 3.093036502832547e-05, 2.9625922252307646e-05, 2.9257877031341195e-05, 2.9086982976878062e-05, 2.8806922273361124e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 0.00017341253987979144, "first_to_last_ratio": 1.4955148931299893, "imbalance_abs_log_ratio": 0.4024705576674927, "last_quartile_mean": 0.00011595507385209203, "mean": 0.00013990553861731314, "normalized": [ 1.3768818258178377, 1.3340879911950096, 1.2910054180628048, 1.2527236427111486, 1.2147542166907819, 1.1769889670731195, 1.1474411155803916, 1.1220953663134672, 1.0944075691302455, 1.0773007485615413, 1.0508678613623372, 1.0320596054419082, 1.0208161736522385, 1.000559648637946, 0.9805197815181176, 0.9663128275349431, 0.948744191771126, 0.9319305822463463, 0.9182019814348666, 0.9075038868226692, 0.8943650364968407, 0.886262676120959, 0.8757907045344768, 0.8679002174812178, 0.856288373659771, 0.8478178092769747, 0.8387215581135022, 0.8313975747674917, 0.8224084044020752, 0.8161864847722404, 0.8112200992010112, 0.8064376596145924 ], "peak_layer": 1, "peak_normalized": 1.3768818258178377, "population_cv": 0.16325888721442564, "rest_mean": 0.0001431150389921472, "spike_contrast": 0.8564734877369171, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001225742365932092, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0001926333934534341, 0.00018664629897102714, 0.00018061880837194622, 0.0001752629759721458, 0.00016995084297377616, 0.0001646672753849998, 0.00016053336730692536, 0.00015698735660407692, 0.00015311368042603135, 0.00015072034148033708, 0.00014702223415952176, 0.00014439085498452187, 0.0001428178366040811, 0.00013998383656144142, 0.00013718014815822244, 0.00013519251660909504, 0.00013273456715978682, 0.00013038225006312132, 0.00012846154277212918, 0.00012696482008323073, 0.00012512662215158343, 0.0001239930570591241, 0.00012252797023393214, 0.000121424047392793, 0.00011979948612861335, 0.00011861440725624561, 0.00011734179133782163, 0.0001163171255029738, 0.00011505949078127742, 0.00011418900976423174, 0.00011349418491590768, 0.00011282509512966499 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 0.00014675183047074825, "first_to_last_ratio": 1.2676545313577883, "imbalance_abs_log_ratio": 0.23716836728277432, "last_quartile_mean": 0.00011576642282307148, "mean": 0.00013041823922321782, "normalized": [ 1.1700698937843759, 1.1560440994564782, 1.1417398043557874, 1.130463200949706, 1.118403537277814, 1.1065304334184665, 1.0939881912841023, 1.0846815122144253, 1.0709468270677047, 1.0579907390333845, 1.0442838370166418, 1.035550673160169, 1.025959021745254, 1.0140203095307772, 1.0062047700154308, 0.9976631870190885, 0.9861765922450596, 0.9770016877798172, 0.9687702939454803, 0.9602422119882037, 0.9504995944091904, 0.9425883202531964, 0.9330014667281973, 0.9259385264566372, 0.9147443798098084, 0.906202796813466, 0.8958065504175365, 0.8885655303249116, 0.8806194435723184, 0.8757905905296257, 0.8713783728577705, 0.8681336045391749 ], "peak_layer": 1, "peak_normalized": 1.1700698937843759, "population_cv": 0.09121429133712369, "rest_mean": 0.0001320278307604086, "spike_contrast": 0.921975648780332, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001217264449223876, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0001525984553154558, 0.0001507692359155044, 0.000148903694935143, 0.00014743302017450333, 0.00014586022007279098, 0.00014431175077334046, 0.00014267601363826543, 0.00014146225294098258, 0.000139670999487862, 0.00013798128929920495, 0.00013619365927297622, 0.00013505469541996717, 0.00013380376913119107, 0.00013224674330558628, 0.00013122745440341532, 0.00013011347618885338, 0.00012861541472375393, 0.00012741883983835578, 0.0001263453159481287, 0.0001252330985153094, 0.0001239624834852293, 0.00012293070903979242, 0.00012168040848337114, 0.00012075927224941552, 0.00011929935135412961, 0.00011818537313956767, 0.00011682951299007982, 0.00011588515189941972, 0.00011484883725643158, 0.00011421906674513593, 0.00011364363308530301, 0.00011322045611450449 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00030004180916876066, "first_to_last_ratio": 2.2837506327398005, "imbalance_abs_log_ratio": 0.8258191057718037, "last_quartile_mean": 0.00013138116082700435, "mean": 0.00019473525253488333, "normalized": [ 1.9000096511818072, 1.783700741635724, 1.6708066910504589, 1.570366884717585, 1.4783058799917985, 1.376185321596543, 1.3064973297630273, 1.2402700043336092, 1.1767953849206394, 1.1505341943958838, 1.0771542571420398, 1.026098162637436, 1.013898063759585, 0.9669473025647412, 0.9095398902231254, 0.8755205777387188, 0.842301214161826, 0.8073124727024382, 0.7791991818813049, 0.760574607765185, 0.7402055397447304, 0.7308143413325056, 0.7152667122155245, 0.7043714155678553, 0.6942467749765324, 0.6868749151283189, 0.680655564574882, 0.676363713518914, 0.6701061029149729, 0.6672276319278244, 0.6636177363814285, 0.6582317375530352 ], "peak_layer": 1, "peak_normalized": 1.9000096511818072, "population_cv": 0.35930746026536914, "rest_mean": 0.00020494150898971216, "spike_contrast": 0.6812747128050888, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013962146767880766, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0003699988592416048, 0.0003473494143690914, 0.0003253649629186839, 0.0003058057918678969, 0.0002878782688640058, 0.0002679917961359024, 0.0002544210874475539, 0.00024152429250534624, 0.00022916354646440595, 0.000224049566895701, 0.00020975990628357977, 0.00019981748482678086, 0.000197441695490852, 0.0001882987271528691, 0.00017711948021315038, 0.00017049472080543637, 0.00016402573965024203, 0.00015721219824627042, 0.00015173754945863038, 0.00014811068831477314, 0.00014414411270990968, 0.0001423153153154999, 0.0001392876438330859, 0.00013716594548895955, 0.00013519432104658335, 0.00013375876005738974, 0.00013254763325676322, 0.0001317118585575372, 0.00013049328117631376, 0.00012993274140171707, 0.00012922976748086512, 0.00012818092363886535 ] } } }, "post_mlp_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "7fd9c911e93175896ad618842c120e77b3f017c17b7eec9c2bfb01fc6dbc8ac2", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.0034885562490671873, 0.004831995815038681, 0.005952076520770788, 0.006925336085259914, 0.00385080324485898, 0.005542940925806761, 0.006729288958013058, 0.007927131839096546, 0.004081308841705322, 0.005811207927763462, 0.007534808944910765, 0.008581922389566898, 0.00499920267611742, 0.007102519739419222, 0.008542952127754688, 0.010447698645293713, 0.005041283555328846, 0.007636935915797949, 0.009387043304741383, 0.011020516976714134, 0.005807018838822842, 0.008367395959794521, 0.01024632342159748, 0.012378672137856483, 0.006453148555010557, 0.009127611294388771, 0.011037957854568958, 0.012611051090061665, 0.007010402157902718, 0.009794782847166061, 0.012445803731679916, 0.014087481424212456 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.00015858902861509705, "first_to_last_ratio": 3.757869257587208, "imbalance_abs_log_ratio": 1.3238521099622325, "last_quartile_mean": 4.220184837322449e-05, "mean": 8.57325238712292e-05, "normalized": [ 2.5462257183789148, 2.3032637538343064, 2.0924289533379263, 1.8886492391629932, 1.7031256747388024, 1.5374176267930122, 1.4224231993460965, 1.3049597038569594, 1.2055618292025094, 1.169245405308266, 1.0769812048488787, 1.0144334047898542, 0.9913512297333369, 0.9317090592952362, 0.8577499774884012, 0.81555804918997, 0.7682725064221458, 0.7217807351250844, 0.6783324192661732, 0.6511800118936032, 0.6192916242842709, 0.6053728292385717, 0.5848720876072855, 0.5718124973482859, 0.5462302773879167, 0.5275173250132701, 0.5112998770983732, 0.49691833866078555, 0.47721658788927634, 0.46551989467389665, 0.46012635628943105, 0.45317260249616614 ], "peak_layer": 1, "peak_normalized": 2.5462257183789148, "population_cv": 0.5686111305901927, "rest_mean": 9.231303704803361e-05, "spike_contrast": 0.5437775023084314, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.019775271648541e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00021829435718245804, 0.00019746461475733668, 0.0001793892151908949, 0.0001619186659809202, 0.00014601326256524771, 0.00013180669338908046, 0.00012194793089292943, 0.00011187748896190897, 0.00010335585830034688, 0.00010024235962191597, 9.233231685357168e-05, 8.69699360919185e-05, 8.499104296788573e-05, 7.987776916706935e-05, 7.353707042057067e-05, 6.991984992055222e-05, 6.586594099644572e-05, 6.188008410390466e-05, 5.8155150327365845e-05, 5.582730591413565e-05, 5.309343396220356e-05, 5.190014053368941e-05, 5.014256021240726e-05, 4.90229285787791e-05, 4.682970029534772e-05, 4.522539165918715e-05, 4.383502891869284e-05, 4.2602063331287354e-05, 4.0912982512963936e-05, 3.991019548266195e-05, 3.944779382436536e-05, 3.885163096128963e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.00021242368711682502, "first_to_last_ratio": 1.8345658968465297, "imbalance_abs_log_ratio": 0.6068078850437921, "last_quartile_mean": 0.0001157896194854402, "mean": 0.0001511171253696375, "normalized": [ 1.7717451736709768, 1.636674793183293, 1.5242755762025908, 1.4186515254985965, 1.3264266232459345, 1.2453738409548603, 1.1893512719369177, 1.1330135273991528, 1.085860457802012, 1.0628542818802846, 1.023252039669589, 0.9942070699734702, 0.9784308644775869, 0.9531083940389613, 0.9232390376562555, 0.9040072643238835, 0.8853775311228679, 0.8680181295617843, 0.8515367513400655, 0.8395795332993111, 0.8267439067367663, 0.8185133286645813, 0.8091730400239778, 0.800791325997512, 0.7909899739933552, 0.7821072339266707, 0.7753403490721149, 0.7682422073314642, 0.7604971517068638, 0.7549863467424895, 0.7508596466770359, 0.7467718018887739 ], "peak_layer": 1, "peak_normalized": 1.7717451736709768, "population_cv": 0.2714186609524326, "rest_mean": 0.00015645541295978345, "spike_contrast": 0.7816308178118705, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012229037238284946, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0002677410375326872, 0.0002473295899108052, 0.00023034414334688336, 0.00021438254043459892, 0.00020044577831868082, 0.00018819731485564262, 0.00017973134526982903, 0.000171217747265473, 0.0001640921109355986, 0.00016061548376455903, 0.00015463090676348656, 0.00015024171443656087, 0.0001478576596127823, 0.00014403100067283958, 0.00013951722939964384, 0.00013661097909789532, 0.00013379570737015456, 0.0001311724045081064, 0.0001286817860091105, 0.00012687484559137374, 0.0001249351626029238, 0.00012369138130452484, 0.00012227990373503417, 0.00012101328320568427, 0.00011953213106608018, 0.0001181897969217971, 0.0001171672047348693, 0.00011609455395955592, 0.00011492414341773838, 0.0001140913664130494, 0.00011346775136189535, 0.00011285000800853595 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.18837944976985455, "first_to_last_ratio": 1.8345658800697515, "imbalance_abs_log_ratio": 0.6068078758989703, "last_quartile_mean": 0.10268339328467846, "mean": 0.1340121782850474, "normalized": [ 1.7717451751532867, 1.6366747548098712, 1.5242756140248972, 1.4186514377480737, 1.3264266357207033, 1.2453739091726768, 1.1893513017871604, 1.1330134634090196, 1.0858604665419038, 1.0628542749024097, 1.0232520332438115, 0.9942070832344971, 0.9784308561813967, 0.9531084116998004, 0.9232390798282929, 0.9040072094634526, 0.8853775581660035, 0.8680181154203164, 0.8515367049614931, 0.8395794986042229, 0.8267439263823189, 0.8185133396785964, 0.809173050926764, 0.8007913534932437, 0.7909900046670136, 0.782107217826699, 0.7753403703776178, 0.7682422246066585, 0.7604971603471317, 0.7549863443945886, 0.7508596538022533, 0.7467717694238248 ], "peak_layer": 1, "peak_normalized": 1.7717451751532867, "population_cv": 0.27141865758333467, "rest_mean": 0.13874622461972413, "spike_contrast": 0.7816308398662988, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.10844832807779312, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.23743543028831482, 0.21933434903621674, 0.20427149534225464, 0.19011656939983368, 0.17775732278823853, 0.16689527034759521, 0.15938755869865417, 0.15183760225772858, 0.14551852643489838, 0.14243541657924652, 0.13712823390960693, 0.1332358568906784, 0.1311216503381729, 0.12772813439369202, 0.1237252801656723, 0.12114797532558441, 0.1186513751745224, 0.1163249984383583, 0.11411628872156143, 0.11251387745141983, 0.11079375445842743, 0.10969075560569763, 0.10843904316425323, 0.107315793633461, 0.10600229352712631, 0.104811891913414, 0.10390505194664001, 0.10295381397008896, 0.1019158810377121, 0.10117736458778381, 0.10062433779239655, 0.10007651150226593 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 6.873207303215167e-05, "first_to_last_ratio": 2.248757814140914, "imbalance_abs_log_ratio": 0.8103779811585758, "last_quartile_mean": 3.0564462122129044e-05, "mean": 4.6857780716891284e-05, "normalized": [ 1.7400119939228862, 1.6405409364325143, 1.555121247708426, 1.4944054279879166, 1.4327475398601994, 1.3414100714975594, 1.2804101716466747, 1.2499370459040815, 1.192804641985012, 1.1608858830017554, 1.098191983597808, 1.057744066133373, 1.0542717513613309, 1.021869458338179, 0.9775749297615711, 0.9570927445976433, 0.9210091333285576, 0.8815125232807631, 0.847820728512489, 0.8204970180441978, 0.7871838673851995, 0.7759041238100245, 0.7506938949785397, 0.7421070509174572, 0.7072987335050381, 0.6891811912847667, 0.6695407652827761, 0.6616333377893114, 0.6345027924930273, 0.6212629426015038, 0.6200302724102036, 0.614801730639213 ], "peak_layer": 1, "peak_normalized": 1.7400119939228862, "population_cv": 0.32175451551289735, "rest_mean": 4.9004235565748616e-05, "spike_contrast": 0.7196709452950106, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.5266924533061685e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 8.153310045599937e-05, 7.687210745643824e-05, 7.28695304132998e-05, 7.002452184678987e-05, 6.713537004543468e-05, 6.28554989816621e-05, 5.9997179050697014e-05, 5.8569276006892323e-05, 5.589217835222371e-05, 5.439653614303097e-05, 5.1458839152473956e-05, 4.9563539505470544e-05, 4.9400834541302174e-05, 4.788253500009887e-05, 4.58069916930981e-05, 4.4847241952084005e-05, 4.315644400776364e-05, 4.130572051508352e-05, 3.972699778387323e-05, 3.8446669350378215e-05, 3.688568904181011e-05, 3.635714529082179e-05, 3.5175849916413426e-05, 3.477348946034908e-05, 3.3142448955914006e-05, 3.2293501135427505e-05, 3.13731943606399e-05, 3.1002669857116416e-05, 2.9731392714893445e-05, 2.9111002731951885e-05, 2.905324254243169e-05, 2.8808244678657502e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 0.00017190765356644988, "first_to_last_ratio": 1.4894621368061003, "imbalance_abs_log_ratio": 0.3984150727783497, "last_quartile_mean": 0.00011541592721187044, "mean": 0.00013894661265112518, "normalized": [ 1.3739040544627474, 1.3296102750467884, 1.2883342981472405, 1.2499918159027226, 1.2130627732820631, 1.175544413473833, 1.1474083142393565, 1.1199112790671786, 1.092909618037032, 1.0755929927697745, 1.0512861461262244, 1.0312571117882559, 1.0194388253067554, 1.0019061415077835, 0.980344379776871, 0.9647961249933756, 0.9493075664595542, 0.9344817410273759, 0.9198929201794295, 0.9079992279088673, 0.8952920956888009, 0.8869530532606651, 0.8771902552778059, 0.8683889858357474, 0.8579407806493952, 0.8482823463732029, 0.8408658204630546, 0.8330843095877946, 0.8243382853592501, 0.818115422616924, 0.8135611224181157, 0.8090075029660142 ], "peak_layer": 1, "peak_normalized": 1.3739040544627474, "population_cv": 0.16161403917150835, "rest_mean": 0.00014210757001786045, "spike_contrast": 0.8576421569620596, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012187744287075474, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00019089931447524577, 0.00018474484386388212, 0.00017900968668982387, 0.00017368212866131216, 0.0001685509632807225, 0.00016333791427314281, 0.00015942849859129637, 0.00015560787869617343, 0.00015185608936008066, 0.00014945000293664634, 0.00014607264893129468, 0.0001432896824553609, 0.00014164757158141583, 0.0001392114645568654, 0.00013621553080156446, 0.0001340551534667611, 0.00013190307072363794, 0.00012984307250007987, 0.0001278160052606836, 0.00012616341700777411, 0.0001243978040292859, 0.00012323912233114243, 0.00012188261462142691, 0.00012065970804542303, 0.00011920796532649547, 0.00011786595860030502, 0.00011683545744745061, 0.00011575424287002534, 0.00011453901242930442, 0.00011367436673026532, 0.00011304156214464456, 0.00011240885214647278 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 0.00014552358334185556, "first_to_last_ratio": 1.262638843036328, "imbalance_abs_log_ratio": 0.2332038508028458, "last_quartile_mean": 0.00011525352965691127, "mean": 0.00012953498708156985, "normalized": [ 1.1676304712442283, 1.1525836984023852, 1.1391521450612143, 1.1288261090295841, 1.1167600441451297, 1.1054423860149858, 1.094731137320412, 1.082319740349084, 1.067957453000188, 1.0563935463552665, 1.04560972624621, 1.0337959762176818, 1.0235535208863231, 1.0159040893770892, 1.0050628637064216, 0.9950792389307415, 0.9865814183778941, 0.9786230528286792, 0.9704418054120744, 0.9608381133357455, 0.9508407831226554, 0.9429749854457838, 0.9346839821902475, 0.9262276149685831, 0.9170542960948607, 0.9059488475652709, 0.8984648923618161, 0.8908037775288885, 0.8831483920181569, 0.8778579808382188, 0.8744051093280881, 0.8703028022960919 ], "peak_layer": 1, "peak_normalized": 1.1676304712442283, "population_cv": 0.0896642712448568, "rest_mean": 0.00013110964479252557, "spike_contrast": 0.9231344927669946, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012103183544240892, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00015124899800866842, 0.00014929991448298097, 0.00014756005839444697, 0.00014622247545048594, 0.00014465949789155275, 0.00014319346519187093, 0.00014180598373059183, 0.00014019827358424664, 0.00013833785487804562, 0.00013683992438018322, 0.0001354430423816666, 0.00013391274842433631, 0.0001325859921053052, 0.00013159512309357524, 0.00013019080506637692, 0.00012889757636003196, 0.00012779681128449738, 0.00012676592450588942, 0.00012570616672746837, 0.00012446215259842575, 0.00012316714855842292, 0.00012214825255796313, 0.00012107427755836397, 0.00011997888213954866, 0.00011879061639774591, 0.00011735207226593047, 0.0001163826382253319, 0.0001153902558144182, 0.00011439861555118114, 0.00011371332220733166, 0.00011326605454087257, 0.0001127346622524783 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00029742828155576717, "first_to_last_ratio": 2.2781890121807993, "imbalance_abs_log_ratio": 0.8233808344454435, "last_quartile_mean": 0.00013055469935352448, "mean": 0.00019329176166138495, "normalized": [ 1.8891850069388627, 1.7801603926271052, 1.6773544463608039, 1.566500262900107, 1.476456430986117, 1.3754925511314788, 1.3084504582544147, 1.236424592515421, 1.172965600373877, 1.150871569148191, 1.0802294364746856, 1.0319209662753672, 1.0080063247818516, 0.9698985570693867, 0.9067931019504507, 0.8756163462587222, 0.8444435806568995, 0.8113939671049883, 0.7809677247391495, 0.7613442367276676, 0.7402582678385915, 0.7293562882295931, 0.7162426695419845, 0.7062419232587185, 0.6940340549797547, 0.6866100033591334, 0.6831688143117768, 0.6753320518823956, 0.6715919825156708, 0.6686410475337363, 0.6640239865195754, 0.6600233567535221 ], "peak_layer": 1, "peak_normalized": 1.8891850069388627, "population_cv": 0.357915029863326, "rest_mean": 0.00020341356920888993, "spike_contrast": 0.6815376252628051, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001386340009048581, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00036516389809548855, 0.0003440903383307159, 0.0003242187958676368, 0.0003027915954589844, 0.00028538686456158757, 0.00026587137836031616, 0.0002529126941226423, 0.00023899068764876574, 0.00022672458726447076, 0.00022245399304665625, 0.0002087994507746771, 0.00019946182146668434, 0.00019483931828290224, 0.00018747340072877705, 0.00017527563613839447, 0.00016924942610785365, 0.00016322398732881993, 0.00015683576930314302, 0.0001509546273155138, 0.00014716156874783337, 0.0001430858246749267, 0.0001409785618307069, 0.00013844380737282336, 0.00013651074550580233, 0.00013415106514003128, 0.00013271605712361634, 0.00013205090363044292, 0.00013053612201474607, 0.00012981319741811603, 0.00012924280599690974, 0.00012835036613978446, 0.00012757707736454904 ] } } }, "pre_attention_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "de71d4cf2d509a1063f6eac7f50c4885e9a058dee7e4b993ad40f5527f528d00", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.02802751213312149, 0.01413465291261673, 0.014225807972252369, 0.01433455292135477, 0.01440478302538395, 0.009703479707241058, 0.00977881159633398, 0.009858808480203152, 0.009950309991836548, 0.00752287870272994, 0.007571155205368996, 0.007693085819482803, 0.007788818795233965, 0.006353048607707024, 0.006389216054230928, 0.0064707002602517605, 0.00656962301582098, 0.005553483963012695, 0.0056249531917274, 0.005688921548426151, 0.005768724251538515, 0.004966245498508215, 0.005057469941675663, 0.005098348017781973, 0.005216081161051989, 0.0046758754178881645, 0.004727107938379049, 0.004777040798217058, 0.004837613552808762, 0.0044186655431985855, 0.004510571248829365, 0.0045849340967834 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.0001580166108396952, "first_to_last_ratio": 1.9713740353643698, "imbalance_abs_log_ratio": 0.6787307795042399, "last_quartile_mean": 8.015557068574708e-05, "mean": 0.00011843875881822896, "normalized": [ 0.8918679597819287, 1.5814819593588525, 1.5722239987443252, 1.3246067564285042, 1.1252650527356378, 1.512743589128775, 1.3788112428745087, 1.2863035124436846, 1.097095055843533, 1.468154353173571, 1.1523254606068964, 1.0839888571909224, 1.054518217028451, 1.146335080549274, 1.1665208527784754, 0.8931136212023528, 0.8203804970547371, 1.073741223277434, 0.919654742394771, 0.8223971330190447, 0.7711288002244273, 0.8470775332584999, 0.8298711629738247, 0.7662487463744663, 0.7210382462310316, 0.7071376059209309, 0.7189559392079661, 0.727980518763353, 0.6111916361855597, 0.6986829946379066, 0.6244678178312321, 0.6046898327751223 ], "peak_layer": 2, "peak_normalized": 1.5814819593588525, "population_cv": 0.2904715201464359, "rest_mean": 0.00012310891098523926, "spike_contrast": 0.7572151875143335, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.321993711637333e-05, "top_five_layers": [ 2, 3, 6, 10, 7 ], "top_five_spike_overlap": 0 }, "values": [ 0.00010563173418631777, 0.0001873087603598833, 0.00018621225899551064, 0.00015688478015363216, 0.0001332749961875379, 0.00017916747310664505, 0.00016330469225067645, 0.00015234819147735834, 0.00012993857671972364, 0.00017388637934345752, 0.0001364799973089248, 0.0001283862948184833, 0.00012489582877606153, 0.00013577050413005054, 0.00013816128193866462, 0.00010577926877886057, 9.71648478298448e-05, 0.00012717257777694613, 0.00010892276623053476, 9.74036956904456e-05, 9.133153798757121e-05, 0.0001003268116619438, 9.828891052166e-05, 9.07535504666157e-05, 8.539887494407594e-05, 8.375250035896897e-05, 8.515224908478558e-05, 8.622110908618197e-05, 7.238877878990024e-05, 8.275114669231698e-05, 7.396119326585904e-05, 7.161871326388791e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.0001664285200604354, "first_to_last_ratio": 1.9719337364651075, "imbalance_abs_log_ratio": 0.6790146534176558, "last_quartile_mean": 8.439863722742302e-05, "mean": 0.0001245867656507471, "normalized": [ 0.8984334461398115, 1.5927952483854981, 1.5658595485525297, 1.3198735865176374, 1.131213894321221, 1.5125315092450449, 1.3796810696180641, 1.286366052942558, 1.0960606282761096, 1.4536884557404346, 1.1543342746982923, 1.0797559630746076, 1.0497530570909623, 1.1457182987306291, 1.1551962694199982, 0.9023721663305979, 0.8238710779144349, 1.0632503993742801, 0.9211624832492573, 0.8274335806257342, 0.7739032412019898, 0.850990856837025, 0.8281317613098801, 0.7681943949939335, 0.7208251570035091, 0.7089546837555952, 0.7203162530736348, 0.7236174125342815, 0.6145740290254166, 0.6994991973458914, 0.6274381906330746, 0.6042038120380651 ], "peak_layer": 2, "peak_normalized": 1.5927952483854981, "population_cv": 0.2889665774838781, "rest_mean": 0.00012946851159169995, "spike_contrast": 0.7586812913967177, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.822533756960183e-05, "top_five_layers": [ 2, 3, 6, 10, 7 ], "top_five_spike_overlap": 0 }, "values": [ 0.00011193291720701382, 0.0001984412083402276, 0.0001950853766174987, 0.000164438781212084, 0.00014093428035266697, 0.00018844140868168324, 0.00017189000209327787, 0.00016026418597903103, 0.00013655464863404632, 0.00018111034296453, 0.00014381477376446128, 0.0001345233031315729, 0.00013078533811494708, 0.00014274133718572557, 0.00014392216689884663, 0.00011242362961638719, 0.00010264343291055411, 0.0001324669283349067, 0.00011476465442683548, 0.00010308727360097691, 9.641810174798593e-05, 0.00010602219845168293, 0.00010317425767425448, 9.570685506332666e-05, 8.980527491075918e-05, 8.832637104205787e-05, 8.974187221610919e-05, 9.015315299620852e-05, 7.656779052922502e-05, 8.714834257261828e-05, 7.817049481673166e-05, 7.527579873567447e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.14759047608822584, "first_to_last_ratio": 1.9719337947467823, "imbalance_abs_log_ratio": 0.6790146829732503, "last_quartile_mean": 0.07484555337578058, "mean": 0.1104847895912826, "normalized": [ 0.8984334974418968, 1.5927952138130326, 1.565859646322909, 1.3198735575362357, 1.1312139236939245, 1.5125315007911495, 1.3796810264894532, 1.286366010451999, 1.0960606168617568, 1.4536884922319049, 1.1543342013148132, 1.0797558953237962, 1.0497530983161607, 1.1457182562234747, 1.1551962948455465, 0.9023721940039697, 0.8238711061937815, 1.0632504861804335, 0.9211625151546727, 0.8274335809310764, 0.7739032642161865, 0.8509909070635955, 0.8281317391228538, 0.7681943896830757, 0.7208251689669645, 0.7089546589522648, 0.7203162343708721, 0.7236173971219663, 0.6145740294939214, 0.6994991437375198, 0.6274381319743458, 0.6042038211744466 ], "peak_layer": 2, "peak_normalized": 1.5927952138130326, "population_cv": 0.28896658018674404, "rest_mean": 0.11481397047086998, "spike_contrast": 0.7586813040631772, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.08710721284151077, "top_five_layers": [ 2, 3, 6, 10, 7 ], "top_five_spike_overlap": 0 }, "values": [ 0.09926323592662811, 0.1759796440601349, 0.1730036735534668, 0.14582595229148865, 0.12498193234205246, 0.16711172461509705, 0.15243376791477203, 0.14212387800216675, 0.12109802663326263, 0.16061046719551086, 0.1275363713502884, 0.1192966029047966, 0.11598175019025803, 0.12658444046974182, 0.12763161957263947, 0.09969840198755264, 0.0910252258181572, 0.11747300624847412, 0.10177444666624069, 0.09141882508993149, 0.08550453931093216, 0.09402155131101608, 0.09149596095085144, 0.08487379550933838, 0.0796402171254158, 0.0783287063241005, 0.07958398759365082, 0.07994871586561203, 0.06790108233690262, 0.07728401571512222, 0.069322369992733, 0.06675533205270767 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 5.093040158499207e-05, "first_to_last_ratio": 0.8860093039525244, "imbalance_abs_log_ratio": 0.12102782735710801, "last_quartile_mean": 5.748291960117058e-05, "mean": 5.7093861016710434e-05, "normalized": [ 0.4337136137178416, 0.9185569667519601, 0.9444095926066538, 0.8728787927927439, 0.7849140301431445, 1.1296292591692207, 1.0726593944235638, 0.9796135302182263, 0.9386642177229508, 1.3046353648934683, 1.0461832118506482, 1.0032917361864164, 0.9662905330123907, 1.0760080966276753, 1.1828739682986207, 0.9087955578115051, 0.9194941496879511, 1.198292373503509, 1.0946132296674007, 1.0673695485587213, 0.9244260207272295, 1.0525679458300632, 1.125653431968099, 0.9999504891362015, 0.9629027708994021, 1.0063234987710272, 0.9975933861195996, 1.0791525148711678, 0.9412468232151083, 1.085173475571988, 1.0031556955576197, 0.9789667796878816 ], "peak_layer": 10, "peak_normalized": 1.3046353648934683, "population_cv": 0.1441687540611157, "rest_mean": 5.695535414647397e-05, "spike_contrast": 1.0155638391297448, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.784179811598733e-05, "top_five_layers": [ 10, 18, 15, 6, 23 ], "top_five_spike_overlap": 1 }, "values": [ 2.4762384782661684e-05, 5.2443963795667514e-05, 5.3919990023132414e-05, 4.9836020480142906e-05, 4.481377254705876e-05, 6.449489592341706e-05, 6.124226638348773e-05, 5.592991874436848e-05, 5.3591964388033375e-05, 7.448667020071298e-05, 5.973063889541663e-05, 5.728179894504137e-05, 5.516925739357248e-05, 6.143345672171563e-05, 6.75348419463262e-05, 5.1886647270293906e-05, 5.249747118796222e-05, 6.84151382301934e-05, 6.249569560168311e-05, 6.094024865888059e-05, 5.277905074763112e-05, 6.009516800986603e-05, 6.426790059776977e-05, 5.7091034250333905e-05, 5.4975836974335834e-05, 5.74548939766828e-05, 5.695645813830197e-05, 6.161298369988799e-05, 5.373941530706361e-05, 6.19567435933277e-05, 5.727403186028823e-05, 5.58929932594765e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 5.801741053801379e-05, "first_to_last_ratio": 0.954195857595535, "imbalance_abs_log_ratio": 0.046886327142486896, "last_quartile_mean": 6.08024129178375e-05, "mean": 6.296396202287724e-05, "normalized": [ 0.4796364558374551, 0.9568353419130547, 0.9804638447804884, 0.8949545354119591, 0.8237377234766136, 1.1597846916175047, 1.085649359633547, 0.9904450076993521, 0.9595078978373608, 1.2941748206155441, 1.0680753760655664, 1.0231559828609242, 0.9904019047543832, 1.096490963646882, 1.1850351594596897, 0.9422215946487361, 0.9418631353047864, 1.1705024179981143, 1.0926353878857977, 1.0382275689133567, 0.9320833313178482, 1.0784583323883348, 1.0985080196909318, 0.9917906747078287, 0.9476388143731139, 0.9855066582268529, 0.978396290111904, 1.028407839812513, 0.8982055143685478, 1.0282804954549911, 0.9451124964564701, 0.9138123627295472 ], "peak_layer": 10, "peak_normalized": 1.2941748206155441, "population_cv": 0.13431233496292436, "rest_mean": 6.285090866084729e-05, "spike_contrast": 1.0115120295379672, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.357445017783903e-05, "top_five_layers": [ 10, 15, 18, 6, 23 ], "top_five_spike_overlap": 1 }, "values": [ 3.019981159013696e-05, 6.0246144130360335e-05, 6.173388828756288e-05, 5.634988337988034e-05, 5.186579073779285e-05, 7.302463927771896e-05, 6.835678505012766e-05, 6.236234185053036e-05, 6.041441884008236e-05, 8.148637425620109e-05, 6.725025741616264e-05, 6.442195444833487e-05, 6.235962791834027e-05, 6.903941539349034e-05, 7.461450877599418e-05, 5.9326004702597857e-05, 5.930343468207866e-05, 7.369946979451925e-05, 6.879665306769311e-05, 6.537092122016475e-05, 5.8687659475253895e-05, 6.790400948375463e-05, 6.916641723364592e-05, 6.244707037694752e-05, 5.966709431959316e-05, 6.205140380188823e-05, 6.160370685392991e-05, 6.475263216998428e-05, 5.655457789544016e-05, 6.474461406469345e-05, 5.950802733423188e-05, 5.753724690293893e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 1.7888660295284353e-05, "first_to_last_ratio": 0.41855462748958633, "imbalance_abs_log_ratio": 0.8709478659353102, "last_quartile_mean": 4.2739129185065394e-05, "mean": 3.1986895351110434e-05, "normalized": [ 0.2593955501951102, 0.4997829486650695, 0.48480994473028793, 0.4968760363401999, 0.5474125689219601, 0.7788406177150305, 0.7554473604850604, 0.6514322393717875, 0.7578135841604356, 0.9674810188795085, 0.9032772556875033, 0.9019220651946442, 0.8311356963419073, 0.9587970177975769, 1.1401394869948454, 0.8953886492354074, 1.0262272842074007, 1.1413916918814533, 1.174037958462901, 1.194927715786966, 1.0267140632187253, 1.3434898267078448, 1.3729067467361273, 1.2011929483563188, 1.1735959904166378, 1.3266745689353452, 1.2739952988364465, 1.4049535244931144, 1.2752889026856997, 1.4680979746684197, 1.4182292846928506, 1.3483241791974143 ], "peak_layer": 30, "peak_normalized": 1.4680979746684197, "population_cv": 0.3184955307507873, "rest_mean": 3.066251991657299e-05, "spike_contrast": 1.276428773763581, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.913852269761264e-05, "top_five_layers": [ 30, 31, 28, 23, 32 ], "top_five_spike_overlap": 1 }, "values": [ 8.297258318634704e-06, 1.5986504877218977e-05, 1.5507564967265353e-05, 1.589352177688852e-05, 1.7510028555989265e-05, 2.491269333404489e-05, 2.416441566310823e-05, 2.083729486912489e-05, 2.4240103812189773e-05, 3.094671410508454e-05, 2.889303505071439e-05, 2.8849686714238487e-05, 2.6585450541460887e-05, 3.066893987124786e-05, 3.6469522456172854e-05, 2.8640703021665104e-05, 3.282582474639639e-05, 3.6509576602838933e-05, 3.755382931558415e-05, 3.822202779701911e-05, 3.2841395295690745e-05, 4.2974068492185324e-05, 4.391502443468198e-05, 3.842243313556537e-05, 3.7539692129939795e-05, 4.243620060151443e-05, 4.0751154301688075e-05, 4.494010136113502e-05, 4.0792532672639936e-05, 4.695989628089592e-05, 4.5364751713350415e-05, 4.3128704419359565e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00022091567916504573, "first_to_last_ratio": 1.232986382535498, "imbalance_abs_log_ratio": 0.20943917994887448, "last_quartile_mean": 0.00017917122386279516, "mean": 0.0002198539991695725, "normalized": [ 0.4672441326905094, 1.0299890566632133, 1.08986441571188, 1.0084939820045402, 0.8814237539069011, 1.2693826047962031, 1.1886514905905403, 1.1035827467935222, 1.0473307057033197, 1.4274044616309502, 1.1442998336482333, 1.0911283607583675, 1.1094038041384326, 1.1946873974128753, 1.2511466100570947, 0.9923165311956169, 0.9239703707694655, 1.2473578196113966, 1.1181336030529359, 1.0160745414007064, 0.9397394354798955, 0.9905446518023159, 1.0121913654482901, 0.9359943959598221, 0.8839446941554127, 0.8510229499852975, 0.8648513533654478, 0.8786898836618169, 0.7455559493639876, 0.8399702482950041, 0.7392957942269532, 0.7163130557190521 ], "peak_layer": 10, "peak_normalized": 1.4274044616309502, "population_cv": 0.18952709251819305, "rest_mean": 0.0002217885959436427, "spike_contrast": 0.9441746799407325, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00020940717658959329, "top_five_layers": [ 10, 6, 15, 18, 14 ], "top_five_spike_overlap": 0 }, "values": [ 0.00010272549116052687, 0.00022644721320830286, 0.00023961105034686625, 0.00022172143508214504, 0.0001937845372594893, 0.0002790788421407342, 0.0002613297838252038, 0.00024262708029709756, 0.00023025984410196543, 0.000313820579322055, 0.00025157889467664063, 0.00023988893372006714, 0.00024390686303377151, 0.000262656802078709, 0.00027506958576850593, 0.0002181647578254342, 0.00020313858112785965, 0.00027423660503700376, 0.0002458261442370713, 0.00022338805138133466, 0.00020660547306761146, 0.00021777520305477083, 0.00022253431961871684, 0.00020578211115207523, 0.0001943387760547921, 0.00018710079893935472, 0.0001901410287246108, 0.00019318348495289683, 0.00016391345707233995, 0.00018467081827111542, 0.00016253713693004102, 0.00015748428995721042 ] } } }, "pre_mlp_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "2999a257265024350fd2665e2a8d452a03cd662193b4245d15290408ad29922d", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.014061377383768559, 0.014165290631353855, 0.014281988143920898, 0.014350161887705326, 0.00966146495193243, 0.009741888381540775, 0.009820535778999329, 0.00991273857653141, 0.0074927168898284435, 0.007546746172010899, 0.00766373984515667, 0.007762390188872814, 0.006325984839349985, 0.006375366821885109, 0.006445344537496567, 0.006555322092026472, 0.005531130358576775, 0.005611324217170477, 0.0056777093559503555, 0.00574630219489336, 0.004946626722812653, 0.0050392793491482735, 0.005084683652967215, 0.005203457549214363, 0.004667261149734259, 0.004717131145298481, 0.004768412560224533, 0.004826710093766451, 0.0044052014127373695, 0.004504534415900707, 0.004574235528707504, 0.004644096828997135 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 3.729895206561196e-05, "first_to_last_ratio": 1.2469721805352876, "imbalance_abs_log_ratio": 0.22071835733540474, "last_quartile_mean": 2.9911615229138988e-05, "mean": 3.340667609563752e-05, "normalized": [ 1.308752778077389, 1.2456464596610868, 1.0236956931411572, 0.9922285647386262, 1.26903181062375, 1.1170846373362797, 1.0287316533628064, 0.9469237624012169, 1.1300261751959062, 1.1132698789554223, 1.0126240729566007, 0.9379202574439188, 1.1130503370514597, 1.070713488608195, 0.9897642722064988, 0.922085688720865, 1.036831181462568, 0.9898105546019274, 0.9395130252451869, 0.9001967292772246, 0.9975952535130206, 0.9404117749145336, 0.9224862220158919, 0.8885787590271026, 0.9461670180110204, 0.8974372095121397, 0.8904691498559152, 0.8684437934704528, 0.9244147277585869, 0.9015689206268677, 0.8740901912628669, 0.8604359589635168 ], "peak_layer": 1, "peak_normalized": 1.308752778077389, "population_cv": 0.1164905978879517, "rest_mean": 3.378375206213807e-05, "spike_contrast": 0.9285666618330386, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.137046587653458e-05, "top_five_layers": [ 1, 5, 2, 9, 6 ], "top_five_spike_overlap": 0 }, "values": [ 4.3721080146497115e-05, 4.161290780757554e-05, 3.419827044126578e-05, 3.314705827506259e-05, 4.239413465256803e-05, 3.73180846509058e-05, 3.4366505133220926e-05, 3.1633575417799875e-05, 3.7750418414361775e-05, 3.719064625329338e-05, 3.3828404411906376e-05, 3.1332798243965954e-05, 3.718331208801828e-05, 3.5768978705164045e-05, 3.306473445263691e-05, 3.080381793552078e-05, 3.463708344497718e-05, 3.3066280593629926e-05, 3.138600732199848e-05, 3.007258055731654e-05, 3.332634150865488e-05, 3.1416031561093405e-05, 3.0817198421573266e-05, 2.9684462788281962e-05, 3.1608295103069395e-05, 2.9980394174344838e-05, 2.9747614462394267e-05, 2.9011820515734144e-05, 3.0881623388268054e-05, 3.01184209092753e-05, 2.9200447897892445e-05, 2.874430538213346e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 6.705871965095866e-05, "first_to_last_ratio": 0.7180297289018083, "imbalance_abs_log_ratio": 0.33124430563536084, "last_quartile_mean": 9.33926785364747e-05, "mean": 8.228966771639534e-05, "normalized": [ 0.8685826408392191, 0.8249006353131848, 0.7154905339803773, 0.6884730939760333, 0.9470487095509537, 0.8638069620260335, 0.8256155016985178, 0.7853664463613442, 0.9851686390305121, 0.9692511243547932, 0.9083804947955202, 0.8729855457896263, 1.0422548534804916, 1.0255492616133721, 0.9871528463964067, 0.9398223255303129, 1.0958375595009802, 1.0726325633807163, 1.0495259774381736, 1.0230838788771643, 1.1576441025653552, 1.1142532603054807, 1.1008567438213033, 1.056908774400763, 1.159063578749664, 1.1192434432436902, 1.1032241584740294, 1.1060435702772593, 1.1786235078649367, 1.1628365878275773, 1.141840825041133, 1.1085318534950752 ], "peak_layer": 29, "peak_normalized": 1.1786235078649367, "population_cv": 0.1341844256762087, "rest_mean": 8.049536754023629e-05, "spike_contrast": 1.1426606459269575, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.197888866765424e-05, "top_five_layers": [ 29, 30, 25, 21, 31 ], "top_five_spike_overlap": 2 }, "values": [ 7.14753768988885e-05, 6.788079917896539e-05, 5.887747829547152e-05, 5.66542221349664e-05, 7.793232362018898e-05, 7.108238787623122e-05, 6.793962529627606e-05, 6.462754390668124e-05, 8.106919995043427e-05, 7.975935295689851e-05, 7.475032907677814e-05, 7.183769048424438e-05, 8.576680556870997e-05, 8.4392107964959e-05, 8.123247971525416e-05, 7.733766688033938e-05, 9.017610864248127e-05, 8.826657722238451e-05, 8.636514394311234e-05, 8.418923243880272e-05, 9.526214853394777e-05, 9.169153054244816e-05, 9.0589135652408e-05, 8.697267185198143e-05, 9.537895675748587e-05, 9.210217103827745e-05, 9.078394941752777e-05, 9.101595787797123e-05, 9.698853682493791e-05, 9.56894364207983e-05, 9.396170207764953e-05, 9.122071787714958e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.05946834245696664, "first_to_last_ratio": 0.7180297222365345, "imbalance_abs_log_ratio": 0.33124431491808765, "last_quartile_mean": 0.08282156102359295, "mean": 0.07297530013602227, "normalized": [ 0.8685826122359729, 0.8249005917343738, 0.71549052911611, 0.6884731350635278, 0.9470486699630107, 0.8638070113577097, 0.8256154769796886, 0.7853664269052657, 0.9851686341386658, 0.9692511556795396, 0.9083805454214465, 0.8729855579949275, 1.0422548040385637, 1.0255492365750694, 0.9871528950273681, 0.9398223256645052, 1.0958375053304956, 1.07263252629438, 1.0495259690486791, 1.0230837911446407, 1.1576441435704576, 1.1142533045483842, 1.1008568124470575, 1.0569088284972943, 1.1590636022344982, 1.119243411839159, 1.1032241423789504, 1.1060435591240758, 1.1786235037173105, 1.1628366076958818, 1.141840809884862, 1.1085318743481278 ], "peak_layer": 29, "peak_normalized": 1.1786235037173105, "population_cv": 0.13418442668798974, "rest_mean": 0.07138409617322462, "spike_contrast": 1.1426607032635474, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.08156780153512955, "top_five_layers": [ 29, 30, 25, 21, 31 ], "top_five_spike_overlap": 2 }, "values": [ 0.06338507682085037, 0.0601973682641983, 0.05221313610672951, 0.050241533666849136, 0.0691111609339714, 0.06303657591342926, 0.060249537229537964, 0.05731235072016716, 0.07189297676086426, 0.07073139399290085, 0.06628934293985367, 0.06370638310909271, 0.07605885714292526, 0.0748397633433342, 0.07203777879476547, 0.06858381628990173, 0.07996907085180283, 0.07827568054199219, 0.07658947259187698, 0.07465984672307968, 0.0844794288277626, 0.08131296932697296, 0.0803353562951088, 0.07712823897600174, 0.08458301424980164, 0.08167712390422821, 0.08050811290740967, 0.08071386069059372, 0.0860104039311409, 0.08485835045576096, 0.08332617580890656, 0.08089544624090195 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 7.408341275549901e-06, "first_to_last_ratio": 0.49124437919492936, "imbalance_abs_log_ratio": 0.7108135577065074, "last_quartile_mean": 1.5080765479069669e-05, "mean": 1.1844782648040564e-05, "normalized": [ 0.5657990507718705, 0.5382233871697658, 0.49942145823009426, 0.48906984415931193, 0.8062921756393574, 0.7122953827323945, 0.7013508394120195, 0.6911626993599347, 0.8940675881372303, 0.868722127857376, 0.9735911331661544, 0.854855860776803, 0.989483587379368, 1.0059088486455374, 1.0303729006301217, 0.9941025538400318, 1.1620912657632405, 1.127012997009919, 1.113401270263767, 1.066097770938846, 1.1826034542812638, 1.1643547935214302, 1.2049778184223279, 1.1791491538213539, 1.2365172470752577, 1.219921447624247, 1.2337761202582145, 1.2288278254433027, 1.3039217543779833, 1.3631114810840155, 1.2856635723507386, 1.3138525898567213 ], "peak_layer": 30, "peak_normalized": 1.3631114810840155, "population_cv": 0.2549101540724733, "rest_mean": 1.1420299651271543e-05, "spike_contrast": 1.237882653019464, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.4136990830593277e-05, "top_five_layers": [ 30, 32, 29, 31, 25 ], "top_five_spike_overlap": 1 }, "values": [ 6.701766778860474e-06, 6.3751390371180605e-06, 5.915538622502936e-06, 5.7929260037781205e-06, 9.550355571263935e-06, 8.436983989668079e-06, 8.307348252856173e-06, 8.186671948351432e-06, 1.0590036254143342e-05, 1.0289824786013924e-05, 1.1531975360412616e-05, 1.0125581866304856e-05, 1.1720218026312068e-05, 1.1914771675947122e-05, 1.220454305439489e-05, 1.1774928680097219e-05, 1.3764718460151926e-05, 1.334922399109928e-05, 1.3187996046326589e-05, 1.2627696378331166e-05, 1.4007680874783546e-05, 1.379152945446549e-05, 1.4272700354922563e-05, 1.3966765436634887e-05, 1.4646278032159898e-05, 1.4449704394792207e-05, 1.4613809980801307e-05, 1.4555198504240252e-05, 1.5444669770658948e-05, 1.614575921848882e-05, 1.5228405572997872e-05, 1.5562298358418047e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 5.6029601637419546e-05, "first_to_last_ratio": 0.6070213107493427, "imbalance_abs_log_ratio": 0.49919138022039955, "last_quartile_mean": 9.230252817360451e-05, "mean": 7.840498903988191e-05, "normalized": [ 0.6598133090619259, 0.6419677920829977, 0.6005256373772865, 0.5913463573822231, 0.8562376513989928, 0.8062968559250673, 0.7923156566726156, 0.768439227921366, 0.9790122729268586, 0.9665730330981341, 0.9163710065350301, 0.8881686280668529, 1.0631190324760253, 1.0511030519715439, 1.0188080221730096, 0.9712812246823179, 1.1361272291219966, 1.1138561419734825, 1.0918134348286874, 1.0639217496772315, 1.203414696914147, 1.1592426077267617, 1.1442664103435203, 1.097952968622282, 1.2042281788778202, 1.1625445130141736, 1.1452986211445626, 1.1473932954411665, 1.2233942816523278, 1.205318760675343, 1.1829469858778916, 1.146901364356358 ], "peak_layer": 29, "peak_normalized": 1.2233942816523278, "population_cv": 0.1898597249088972, "rest_mean": 7.605543874820728e-05, "spike_contrast": 1.1977126437531995, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.109256061492487e-05, "top_five_layers": [ 29, 30, 25, 21, 31 ], "top_five_spike_overlap": 2 }, "values": [ 5.173265526536852e-05, 5.033347770222463e-05, 4.708420601673424e-05, 4.636450466932729e-05, 6.713330367347226e-05, 6.321769615169615e-05, 6.212150037754327e-05, 6.024946924299002e-05, 7.675944652874023e-05, 7.578414806630462e-05, 7.184805872384459e-05, 6.963685154914856e-05, 8.335383608937263e-05, 8.241172326961532e-05, 7.987963181221858e-05, 7.615329377586022e-05, 8.907804294722155e-05, 8.733187860343605e-05, 8.560362039133906e-05, 8.341677312273532e-05, 9.435371612198651e-05, 9.089040395338088e-05, 8.971619536168873e-05, 8.608499047113582e-05, 9.441749716643244e-05, 9.114928980125114e-05, 8.97971258382313e-05, 8.996135875349864e-05, 9.592021524440497e-05, 9.450300422031432e-05, 9.274894546251744e-05, 8.992278890218586e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 4.8511469685763586e-05, "first_to_last_ratio": 0.5271352332510719, "imbalance_abs_log_ratio": 0.6402981537748061, "last_quartile_mean": 9.202850924339145e-05, "mean": 7.532146378252946e-05, "normalized": [ 0.548104082267993, 0.5416830199005969, 0.528884702689083, 0.5247448675895723, 0.7777813723908718, 0.7487613756360456, 0.7496059866168319, 0.7329069161583202, 0.9480676391902623, 0.9399123881532756, 0.9000936195631334, 0.883441930780847, 1.0560589316963704, 1.0534552095740957, 1.032095781768693, 0.9835620691098637, 1.1601754560133901, 1.1467788545782247, 1.1233340534265408, 1.0967375255670042, 1.2396083970871608, 1.1951467072205837, 1.1828066060767042, 1.1317734013234224, 1.244698956870482, 1.2025862582796845, 1.1881970083126314, 1.1882256015362047, 1.2689460104605979, 1.2589162611011824, 1.2307315535498626, 1.1921774555104672 ], "peak_layer": 29, "peak_normalized": 1.2689460104605979, "population_cv": 0.23486334953575583, "rest_mean": 7.254842300123225e-05, "spike_contrast": 1.244629176846487, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.029588400153444e-05, "top_five_layers": [ 29, 30, 25, 21, 31 ], "top_five_spike_overlap": 2 }, "values": [ 4.1284001781605184e-05, 4.080035796505399e-05, 3.9836369978729635e-05, 3.952455153921619e-05, 5.858363147126511e-05, 5.639780283672735e-05, 5.6461420172126964e-05, 5.520362174138427e-05, 7.140984234865755e-05, 7.079557690303773e-05, 6.77963689668104e-05, 6.654213939327747e-05, 7.954390457598493e-05, 7.934778841445222e-05, 7.773896504659206e-05, 7.408333476632833e-05, 8.738611359149218e-05, 8.637706196168438e-05, 8.461116522084922e-05, 8.26078758109361e-05, 9.336911898571998e-05, 9.002019942272455e-05, 8.909072494134307e-05, 8.524682925781235e-05, 9.375254740007222e-05, 9.058055729838088e-05, 8.949673792812973e-05, 8.949889161158353e-05, 9.557887096889317e-05, 9.482341556577012e-05, 9.270050213672221e-05, 8.979655103757977e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 9.61669284151867e-05, "first_to_last_ratio": 0.8297833624837898, "imbalance_abs_log_ratio": 0.18659062131596024, "last_quartile_mean": 0.00011589401856326731, "mean": 0.00010716335873439675, "normalized": [ 0.9380742209455786, 0.8898820858744699, 0.7837417114644826, 0.7583102664636657, 1.0822070619439073, 0.9520749081713696, 0.9107982467806989, 0.8640017807748708, 1.0669538303069324, 1.0266950649381863, 0.9594029855154438, 0.8901506143407661, 1.0795289063552718, 1.0379165674077444, 0.9706971366471856, 0.914979822300863, 1.052939971267624, 1.0225707935964177, 0.9851238030379684, 0.9636652893153618, 1.0990044510799908, 1.0523904893769196, 1.0422234788441622, 1.0049019599938585, 1.0962247910521359, 1.0599378706264646, 1.05039930496957, 1.0583562345651827, 1.1199788667078585, 1.1100529546792268, 1.0850973923636869, 1.0717171382921347 ], "peak_layer": 29, "peak_normalized": 1.1199788667078585, "population_cv": 0.09156717959150794, "rest_mean": 0.00010599351123716958, "spike_contrast": 1.0706366257223145, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00011348053521942347, "top_five_layers": [ 29, 30, 21, 25, 31 ], "top_five_spike_overlap": 2 }, "values": [ 0.00010052718425868079, 9.536275319987908e-05, 8.398839418077841e-05, 8.12630751170218e-05, 0.00011597294360399246, 0.00010202754492638633, 9.760419925441965e-05, 9.25893327803351e-05, 0.00011433835607022047, 0.00011002409155480564, 0.00010281284630764276, 9.539152961224318e-05, 0.000115685943455901, 0.0001112266254494898, 0.0001040231654769741, 9.805231093196198e-05, 0.00011283658386673778, 0.00010958212078548968, 0.00010556917550275102, 0.00010326960909878835, 0.00011777300824178383, 0.0001127776995417662, 0.00011168816854478791, 0.00010768866923172027, 0.00011747513053705916, 0.00011358650226611644, 0.00011256431753281504, 0.00011341700883349404, 0.00012002069706795737, 0.00011895700299646705, 0.00011628268111962825, 0.00011484880815260112 ] } } } }, "selector": { "expected_uniform_count": 8, "identities_unique": true, "identity_order_sha256": "97fd9721a263024f6a746638f32e0ad9a988cd4c111c00e9e178c1022e5c9d5e", "passed": true, "uniform_count": 8, "uniform_indices": [ 40, 41, 42, 43, 44, 45, 46, 47 ], "visit_count": 65, "visits": [ { "branch": "attention", "coefficient": "detached_learned", "group": 1, "index": 0, "kind": "depth", "layer": 1, "offset": 1, "sources": 1, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 1, "index": 1, "kind": "depth", "layer": 1, "offset": 1, "sources": 2, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 1, "index": 2, "kind": "depth", "layer": 2, "offset": 2, "sources": 2, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 1, "index": 3, "kind": "depth", "layer": 2, "offset": 2, "sources": 2, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 1, "index": 4, "kind": "depth", "layer": 3, "offset": 3, "sources": 2, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 1, "index": 5, "kind": "depth", "layer": 3, "offset": 3, "sources": 2, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 1, "index": 6, "kind": "depth", "layer": 4, "offset": 4, "sources": 2, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 1, "index": 7, "kind": "depth", "layer": 4, "offset": 4, "sources": 2, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 2, "index": 8, "kind": "depth", "layer": 5, "offset": 1, "sources": 2, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 2, "index": 9, "kind": "depth", "layer": 5, "offset": 1, "sources": 3, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 2, "index": 10, "kind": "depth", "layer": 6, "offset": 2, "sources": 3, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 2, "index": 11, "kind": "depth", "layer": 6, "offset": 2, "sources": 3, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 2, "index": 12, "kind": "depth", "layer": 7, "offset": 3, "sources": 3, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 2, "index": 13, "kind": "depth", "layer": 7, "offset": 3, "sources": 3, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 2, "index": 14, "kind": "depth", "layer": 8, "offset": 4, "sources": 3, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 2, "index": 15, "kind": "depth", "layer": 8, "offset": 4, "sources": 3, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 3, "index": 16, "kind": "depth", "layer": 9, "offset": 1, "sources": 3, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 3, "index": 17, "kind": "depth", "layer": 9, "offset": 1, "sources": 4, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 3, "index": 18, "kind": "depth", "layer": 10, "offset": 2, "sources": 4, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 3, "index": 19, "kind": "depth", "layer": 10, "offset": 2, "sources": 4, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 3, "index": 20, "kind": "depth", "layer": 11, "offset": 3, "sources": 4, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 3, "index": 21, "kind": "depth", "layer": 11, "offset": 3, "sources": 4, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 3, "index": 22, "kind": "depth", "layer": 12, "offset": 4, "sources": 4, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 3, "index": 23, "kind": "depth", "layer": 12, "offset": 4, "sources": 4, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 4, "index": 24, "kind": "depth", "layer": 13, "offset": 1, "sources": 4, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 4, "index": 25, "kind": "depth", "layer": 13, "offset": 1, "sources": 5, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 4, "index": 26, "kind": "depth", "layer": 14, "offset": 2, "sources": 5, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 4, "index": 27, "kind": "depth", "layer": 14, "offset": 2, "sources": 5, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 4, "index": 28, "kind": "depth", "layer": 15, "offset": 3, "sources": 5, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 4, "index": 29, "kind": "depth", "layer": 15, "offset": 3, "sources": 5, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 4, "index": 30, "kind": "depth", "layer": 16, "offset": 4, "sources": 5, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 4, "index": 31, "kind": "depth", "layer": 16, "offset": 4, "sources": 5, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 5, "index": 32, "kind": "depth", "layer": 17, "offset": 1, "sources": 5, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 5, "index": 33, "kind": "depth", "layer": 17, "offset": 1, "sources": 6, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 5, "index": 34, "kind": "depth", "layer": 18, "offset": 2, "sources": 6, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 5, "index": 35, "kind": "depth", "layer": 18, "offset": 2, "sources": 6, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 5, "index": 36, "kind": "depth", "layer": 19, "offset": 3, "sources": 6, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 5, "index": 37, "kind": "depth", "layer": 19, "offset": 3, "sources": 6, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 5, "index": 38, "kind": "depth", "layer": 20, "offset": 4, "sources": 6, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 5, "index": 39, "kind": "depth", "layer": 20, "offset": 4, "sources": 6, "use_uniform": false }, { "branch": "attention", "coefficient": "uniform", "group": 6, "index": 40, "kind": "depth", "layer": 21, "offset": 1, "sources": 6, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 6, "index": 41, "kind": "depth", "layer": 21, "offset": 1, "sources": 7, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 6, "index": 42, "kind": "depth", "layer": 22, "offset": 2, "sources": 7, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 6, "index": 43, "kind": "depth", "layer": 22, "offset": 2, "sources": 7, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 6, "index": 44, "kind": "depth", "layer": 23, "offset": 3, "sources": 7, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 6, "index": 45, "kind": "depth", "layer": 23, "offset": 3, "sources": 7, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 6, "index": 46, "kind": "depth", "layer": 24, "offset": 4, "sources": 7, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 6, "index": 47, "kind": "depth", "layer": 24, "offset": 4, "sources": 7, "use_uniform": true }, { "branch": "attention", "coefficient": "detached_learned", "group": 7, "index": 48, "kind": "depth", "layer": 25, "offset": 1, "sources": 7, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 7, "index": 49, "kind": "depth", "layer": 25, "offset": 1, "sources": 8, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 7, "index": 50, "kind": "depth", "layer": 26, "offset": 2, "sources": 8, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 7, "index": 51, "kind": "depth", "layer": 26, "offset": 2, "sources": 8, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 7, "index": 52, "kind": "depth", "layer": 27, "offset": 3, "sources": 8, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 7, "index": 53, "kind": "depth", "layer": 27, "offset": 3, "sources": 8, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 7, "index": 54, "kind": "depth", "layer": 28, "offset": 4, "sources": 8, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 7, "index": 55, "kind": "depth", "layer": 28, "offset": 4, "sources": 8, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 8, "index": 56, "kind": "depth", "layer": 29, "offset": 1, "sources": 8, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 8, "index": 57, "kind": "depth", "layer": 29, "offset": 1, "sources": 9, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 8, "index": 58, "kind": "depth", "layer": 30, "offset": 2, "sources": 9, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 8, "index": 59, "kind": "depth", "layer": 30, "offset": 2, "sources": 9, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 8, "index": 60, "kind": "depth", "layer": 31, "offset": 3, "sources": 9, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 8, "index": 61, "kind": "depth", "layer": 31, "offset": 3, "sources": 9, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 8, "index": 62, "kind": "depth", "layer": 32, "offset": 4, "sources": 9, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 8, "index": 63, "kind": "depth", "layer": 32, "offset": 4, "sources": 9, "use_uniform": false }, { "branch": "output", "coefficient": "detached_learned", "group": null, "index": 64, "kind": "output", "layer": null, "offset": null, "sources": 9, "use_uniform": false } ], "visits_sha256": "a2084e8fee434b0287d854d9862274a543396a9504e35b643f6497973bcbbd04" } }, "uniform_group_7_attention_only": { "bits_per_byte": 7.959341947457508, "forward": { "activation_sha256": { "attention_branch_output": "676759a2d7c0615ad7e1f246e90927fb829d00c8b6c3745032ae6b27ab7d902a", "mlp_branch_output": "afe29f3ef4d2992353f0bb59fe7c5c2c5050fca671a53184825a8d63cd83a2bc", "post_attention_state": "31853ffb07b24cc59e774bb20656840b4358b3080b42516d380f2dc0f46404c8", "post_mlp_state": "7fd9c911e93175896ad618842c120e77b3f017c17b7eec9c2bfb01fc6dbc8ac2", "pre_attention_input": "de71d4cf2d509a1063f6eac7f50c4885e9a058dee7e4b993ad40f5527f528d00", "pre_mlp_input": "2999a257265024350fd2665e2a8d452a03cd662193b4245d15290408ad29922d" }, "logits_sha256": "873a309a197bfd08fa523edcb440d2697903a34abc02dad1c7f5991509c3c151", "loss_nats": 5.516995429992676, "mixer_summary_sha256": "8450b24bdc777885a3357dc7b8080e7410acbe96943686433da76f379295133a" }, "loss_nats": 5.516995429992676, "loss_scale": 1.0, "mixers": [ { "branch": "attention", "entropy_mean": -0.0, "group": 1, "latest_source_mass": 1.0, "layer": 1, "max_source_mass": 1.0, "mixer_index": 0, "normalized_entropy": 1.0, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 1.0, "median": 1.0, "p05": 1.0, "p95": 1.0 } ], "sources": 1 }, { "branch": "mlp", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 1, "max_source_mass": 0.5, "mixer_index": 1, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 2, "max_source_mass": 0.5, "mixer_index": 2, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 2, "max_source_mass": 0.5, "mixer_index": 3, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 3, "max_source_mass": 0.5, "mixer_index": 4, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 3, "max_source_mass": 0.5, "mixer_index": 5, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 4, "max_source_mass": 0.5, "mixer_index": 6, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 4, "max_source_mass": 0.5, "mixer_index": 7, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6931471824645996, "group": 2, "latest_source_mass": 0.5, "layer": 5, "max_source_mass": 0.5, "mixer_index": 8, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "completed_group_1", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 5, "max_source_mass": 0.3333333432674408, "mixer_index": 9, "normalized_entropy": 1.0000000180537645, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 6, "max_source_mass": 0.3333333432674408, "mixer_index": 10, "normalized_entropy": 1.0000000180537645, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 6, "max_source_mass": 0.3333333432674408, "mixer_index": 11, "normalized_entropy": 1.0000000180537645, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 7, "max_source_mass": 0.3333333432674408, "mixer_index": 12, "normalized_entropy": 1.0000000180537645, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 7, "max_source_mass": 0.3333333432674408, "mixer_index": 13, "normalized_entropy": 1.0000000180537645, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 8, "max_source_mass": 0.3333333432674408, "mixer_index": 14, "normalized_entropy": 1.0000000180537645, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 8, "max_source_mass": 0.3333333432674408, "mixer_index": 15, "normalized_entropy": 1.0000000180537645, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0986123085021973, "group": 3, "latest_source_mass": 0.3333333432674408, "layer": 9, "max_source_mass": 0.3333333432674408, "mixer_index": 16, "normalized_entropy": 1.0000000180537645, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_2", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 9, "max_source_mass": 0.25, "mixer_index": 17, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 10, "max_source_mass": 0.25, "mixer_index": 18, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 10, "max_source_mass": 0.25, "mixer_index": 19, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 11, "max_source_mass": 0.25, "mixer_index": 20, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 11, "max_source_mass": 0.25, "mixer_index": 21, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 12, "max_source_mass": 0.25, "mixer_index": 22, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 12, "max_source_mass": 0.25, "mixer_index": 23, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3862943649291992, "group": 4, "latest_source_mass": 0.25, "layer": 13, "max_source_mass": 0.25, "mixer_index": 24, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_3", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 13, "max_source_mass": 0.20000000298023224, "mixer_index": 25, "normalized_entropy": 1.0000000186840277, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 14, "max_source_mass": 0.20000000298023224, "mixer_index": 26, "normalized_entropy": 1.0000000186840277, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 14, "max_source_mass": 0.20000000298023224, "mixer_index": 27, "normalized_entropy": 1.0000000186840277, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 15, "max_source_mass": 0.20000000298023224, "mixer_index": 28, "normalized_entropy": 1.0000000186840277, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 15, "max_source_mass": 0.20000000298023224, "mixer_index": 29, "normalized_entropy": 1.0000000186840277, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 16, "max_source_mass": 0.20000000298023224, "mixer_index": 30, "normalized_entropy": 1.0000000186840277, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 16, "max_source_mass": 0.20000000298023224, "mixer_index": 31, "normalized_entropy": 1.0000000186840277, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.6094379425048828, "group": 5, "latest_source_mass": 0.20000000298023224, "layer": 17, "max_source_mass": 0.20000000298023224, "mixer_index": 32, "normalized_entropy": 1.0000000186840277, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_4", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 17, "max_source_mass": 0.1666666716337204, "mixer_index": 33, "normalized_entropy": 1.0000001451965654, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 18, "max_source_mass": 0.1666666716337204, "mixer_index": 34, "normalized_entropy": 1.0000001451965654, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 18, "max_source_mass": 0.1666666716337204, "mixer_index": 35, "normalized_entropy": 1.0000001451965654, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 19, "max_source_mass": 0.1666666716337204, "mixer_index": 36, "normalized_entropy": 1.0000001451965654, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 19, "max_source_mass": 0.1666666716337204, "mixer_index": 37, "normalized_entropy": 1.0000001451965654, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 20, "max_source_mass": 0.1666666716337204, "mixer_index": 38, "normalized_entropy": 1.0000001451965654, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 20, "max_source_mass": 0.1666666716337204, "mixer_index": 39, "normalized_entropy": 1.0000001451965654, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.791759729385376, "group": 6, "latest_source_mass": 0.1666666716337204, "layer": 21, "max_source_mass": 0.1666666716337204, "mixer_index": 40, "normalized_entropy": 1.0000001451965654, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_5", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 21, "max_source_mass": 0.1428571492433548, "mixer_index": 41, "normalized_entropy": 1.00000003408302, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 22, "max_source_mass": 0.1428571492433548, "mixer_index": 42, "normalized_entropy": 1.00000003408302, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 22, "max_source_mass": 0.1428571492433548, "mixer_index": 43, "normalized_entropy": 1.00000003408302, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 23, "max_source_mass": 0.1428571492433548, "mixer_index": 44, "normalized_entropy": 1.00000003408302, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 23, "max_source_mass": 0.1428571492433548, "mixer_index": 45, "normalized_entropy": 1.00000003408302, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 24, "max_source_mass": 0.1428571492433548, "mixer_index": 46, "normalized_entropy": 1.00000003408302, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 24, "max_source_mass": 0.1428571492433548, "mixer_index": 47, "normalized_entropy": 1.00000003408302, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9459102153778076, "group": 7, "latest_source_mass": 0.1428571492433548, "layer": 25, "max_source_mass": 0.1428571492433548, "mixer_index": 48, "normalized_entropy": 1.00000003408302, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_6", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 25, "max_source_mass": 0.125, "mixer_index": 49, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 26, "max_source_mass": 0.125, "mixer_index": 50, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 26, "max_source_mass": 0.125, "mixer_index": 51, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 27, "max_source_mass": 0.125, "mixer_index": 52, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 27, "max_source_mass": 0.125, "mixer_index": 53, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 28, "max_source_mass": 0.125, "mixer_index": 54, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 28, "max_source_mass": 0.125, "mixer_index": 55, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.079441547393799, "group": 8, "latest_source_mass": 0.125, "layer": 29, "max_source_mass": 0.125, "mixer_index": 56, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_7", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 29, "max_source_mass": 0.1111111119389534, "mixer_index": 57, "normalized_entropy": 1.0000000180537645, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 30, "max_source_mass": 0.1111111119389534, "mixer_index": 58, "normalized_entropy": 1.0000000180537645, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 30, "max_source_mass": 0.1111111119389534, "mixer_index": 59, "normalized_entropy": 1.0000000180537645, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 31, "max_source_mass": 0.1111111119389534, "mixer_index": 60, "normalized_entropy": 1.0000000180537645, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 31, "max_source_mass": 0.1111111119389534, "mixer_index": 61, "normalized_entropy": 1.0000000180537645, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 32, "max_source_mass": 0.1111111119389534, "mixer_index": 62, "normalized_entropy": 1.0000000180537645, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 32, "max_source_mass": 0.1111111119389534, "mixer_index": 63, "normalized_entropy": 1.0000000180537645, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "output", "entropy_mean": 2.1972246170043945, "group": null, "latest_source_mass": 0.1111111119389534, "layer": null, "max_source_mass": 0.1111111119389534, "mixer_index": null, "normalized_entropy": 1.0000000180537645, "offset": null, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_8", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 } ], "mode": "uniform_group_7_attention_only", "optimizer_state_after": "b82a349700eda58e2330a22ca2ee1f7fb1406efd47fd9aa306122b368a114192", "optimizer_state_before": "b82a349700eda58e2330a22ca2ee1f7fb1406efd47fd9aa306122b368a114192", "optimizer_state_unchanged": true, "positions": { "attention_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "676759a2d7c0615ad7e1f246e90927fb829d00c8b6c3745032ae6b27ab7d902a", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.0022265110164880753, 0.0021613112185150385, 0.0025601957459002733, 0.0025323182344436646, 0.0027253651060163975, 0.002904455177485943, 0.0029208383057266474, 0.00299624796025455, 0.0030937574338167906, 0.0033804215490818024, 0.0034370317589491606, 0.0038969130255281925, 0.004255065228790045, 0.004119740333408117, 0.003970506135374308, 0.004385598469525576, 0.0041999719105660915, 0.005232699681073427, 0.004664790350943804, 0.004898969549685717, 0.005131533369421959, 0.005069608334451914, 0.0051318020559847355, 0.0058372304774820805, 0.0059690922498703, 0.00554921617731452, 0.0058550527319312096, 0.006255472078919411, 0.006502739619463682, 0.006122888531535864, 0.007071305997669697, 0.006241494789719582 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.00015962477027642308, "first_to_last_ratio": 3.7699040072151497, "imbalance_abs_log_ratio": 1.3270495388586014, "last_quartile_mean": 4.234186599205714e-05, "mean": 8.617327489446325e-05, "normalized": [ 2.5428430303860874, 2.3222064178412967, 2.1011513502014654, 1.8911470095267005, 1.7004447966680725, 1.5353700031893112, 1.4229335846259188, 1.3028620492588128, 1.2049362956920768, 1.1705197150696323, 1.0773612761873523, 1.0133200843661747, 0.9906112963967714, 0.9272530070793641, 0.8550207003106389, 0.8185600560490035, 0.7703816559177782, 0.7173918665339755, 0.6760157258502337, 0.6514535263006792, 0.6184138115448249, 0.6053606047045522, 0.5838821473278404, 0.5697017460729616, 0.5457761369570343, 0.5276086337508958, 0.5104655265174319, 0.4951752820428687, 0.47527333481689216, 0.465789239220051, 0.45896414036514044, 0.45180594922816086 ], "peak_layer": 1, "peak_normalized": 2.5428430303860874, "population_cv": 0.5705341151299013, "rest_mean": 9.280180475562466e-05, "spike_contrast": 0.5428688997682254, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.0379213644191625e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00021912511147093028, 0.00020011213200632483, 0.00018106309289578348, 0.00016296633111778647, 0.00014653289690613747, 0.00013230786134954542, 0.0001226188469445333, 0.00011227188952034339, 0.0001038333066389896, 0.00010086751717608422, 9.283974941354245e-05, 8.732111018616706e-05, 8.536421955795959e-05, 7.990442827576771e-05, 7.367993384832516e-05, 7.053800072753802e-05, 6.63863102090545e-05, 6.182000652188435e-05, 5.825448897667229e-05, 5.6137883802875876e-05, 5.329074338078499e-05, 5.216590579948388e-05, 5.031503678765148e-05, 4.909306517220102e-05, 4.703131708083674e-05, 4.5465763832908124e-05, 4.3988486140733585e-05, 4.2670875700423494e-05, 4.095585973118432e-05, 4.013858415419236e-05, 3.955044303438626e-05, 3.893359826179221e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.00021442661636683624, "first_to_last_ratio": 1.843144421770586, "imbalance_abs_log_ratio": 0.6114730379455638, "last_quartile_mean": 0.0001163373926829081, "mean": 0.00015221274247778638, "normalized": [ 1.7741446897870732, 1.648461275666186, 1.5301464224684427, 1.4217121808823485, 1.3265065496562456, 1.2454420544341354, 1.1897998831173078, 1.1336248235930768, 1.0869089599614004, 1.0643003125599004, 1.0227030055526172, 0.9941765648180964, 0.9788958472869692, 0.9511102301475627, 0.9228060651723192, 0.9052418824754537, 0.8846484396889256, 0.865292475366091, 0.8497311645646503, 0.8388691935254817, 0.8256303537004397, 0.8176325372729673, 0.8076852907347736, 0.8000670639426607, 0.7892588692968658, 0.7814415503461742, 0.7731202151361662, 0.7664994567889658, 0.758490072461763, 0.7529625284402992, 0.7485174912671357, 0.7441725498875049 ], "peak_layer": 1, "peak_normalized": 1.7741446897870732, "population_cv": 0.27364872432853743, "rest_mean": 0.00015762320577373935, "spike_contrast": 0.7803181014868814, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012299624067964033, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00027004742878489196, 0.0002509168116375804, 0.0002329077833564952, 0.00021640271006617695, 0.00020191119983792305, 0.00018957215070258826, 0.0001811027032090351, 0.00017255214333999902, 0.00016544139361940324, 0.00016200006939470768, 0.00015566842921543866, 0.0001513263414381072, 0.00014900042151566595, 0.0001447710965294391, 0.00014046284195501357, 0.0001377893495373428, 0.00013465476513374597, 0.00013170854072086513, 0.00012933991092722863, 0.0001276865805266425, 0.00012567146040964872, 0.00012445409083738923, 0.00012293999316170812, 0.00012178040196886286, 0.00012013525702059269, 0.00011894536146428436, 0.00011767874821089208, 0.00011667098442558199, 0.00011545185407157987, 0.00011461049143690616, 0.00011393390013836324, 0.00011327254469506443 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.19015566632151604, "first_to_last_ratio": 1.8431444167877922, "imbalance_abs_log_ratio": 0.6114730352421437, "last_quartile_mean": 0.10316916275769472, "mean": 0.13498378219082952, "normalized": [ 1.7741447141792424, 1.6484611693507762, 1.5301464164964773, 1.421712099711509, 1.3265065258282553, 1.245442086656075, 1.1897999853143832, 1.1336247998774232, 1.0869090145314644, 1.0643003539468234, 1.0227030148964142, 0.9941765578539046, 0.978895844117059, 0.9511102293531465, 0.9228060470726628, 0.9052419251832852, 0.8846484748200404, 0.8652925213936268, 0.8497311905993837, 0.8388692019864461, 0.8256303029992552, 0.8176324960789627, 0.8076852727084025, 0.800067049482784, 0.789258877083647, 0.7814415614650243, 0.7731201949085273, 0.7664994759042474, 0.758490077799444, 0.7529625178867759, 0.7485174642151933, 0.7441725362993381 ], "peak_layer": 1, "peak_normalized": 1.7741447141792424, "population_cv": 0.2736487184871982, "rest_mean": 0.13978183573042904, "spike_contrast": 0.7803180757143806, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.10907429307699204, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.23948076367378235, 0.2225155234336853, 0.20654495060443878, 0.1919080764055252, 0.17905686795711517, 0.16811448335647583, 0.16060370206832886, 0.1530209630727768, 0.14671508967876434, 0.14366328716278076, 0.13804832100868225, 0.13419771194458008, 0.1321350634098053, 0.12838445603847504, 0.12456385046243668, 0.12219297885894775, 0.11941319704055786, 0.116800457239151, 0.11469992995262146, 0.11323373764753342, 0.11144670099020004, 0.11036712676286697, 0.10902441293001175, 0.10799607634544373, 0.10653714835643768, 0.10548193752765656, 0.10435868799686432, 0.1034649983048439, 0.10238385945558548, 0.10163772851228714, 0.10103771835565567, 0.10045122355222702 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 6.883365585963475e-05, "first_to_last_ratio": 2.2497546817252747, "imbalance_abs_log_ratio": 0.8108211799277664, "last_quartile_mean": 3.059607183786284e-05, "mean": 4.691841348858361e-05, "normalized": [ 1.7359303646705315, 1.6475859007944922, 1.5536416135247637, 1.4921074478374423, 1.436565604934822, 1.343834173349448, 1.2787439414688093, 1.2483315192759759, 1.191227587726315, 1.1603086071531024, 1.0995306735006558, 1.059977946939621, 1.0583984121521266, 1.0176934649371145, 0.9754909410255406, 0.9631621800279858, 0.9242301483750994, 0.8752327792179452, 0.8454160845678106, 0.8232909667825862, 0.7863453139125789, 0.77746685672799, 0.7497386589361914, 0.7388508716017261, 0.7093180450413769, 0.6908354487677191, 0.6685471901846501, 0.6592354484095886, 0.631440334976849, 0.6235901145414114, 0.6199506557967245, 0.6139807028410055 ], "peak_layer": 1, "peak_normalized": 1.7359303646705315, "population_cv": 0.3220412433027962, "rest_mean": 4.907019663530971e-05, "spike_contrast": 0.7193528234378936, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.529878449626267e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 8.144709863699973e-05, 7.730211655143648e-05, 7.289439963642508e-05, 7.000731420703232e-05, 6.740137905580923e-05, 6.305056740529835e-05, 5.999663699185476e-05, 5.856973439222202e-05, 5.5890508519951254e-05, 5.4439839004771784e-05, 5.1588234782684594e-05, 4.9732483603293076e-05, 4.965837433701381e-05, 4.774856279254891e-05, 4.576848732540384e-05, 4.519004141911864e-05, 4.3363412260077894e-05, 4.106453343410976e-05, 3.966558142565191e-05, 3.862750600092113e-05, 3.6894074582960457e-05, 3.647751145763323e-05, 3.517654840834439e-05, 3.4665710700210184e-05, 3.328007733216509e-05, 3.241290323785506e-05, 3.136717350571416e-05, 3.0930281354812905e-05, 2.9626178729813546e-05, 2.925785884144716e-05, 2.9087101211189292e-05, 2.8807000489905477e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 0.0001734125216898974, "first_to_last_ratio": 1.495517223041696, "imbalance_abs_log_ratio": 0.4024721155990803, "last_quartile_mean": 0.00011595488103921525, "mean": 0.00013990532806928968, "normalized": [ 1.3768889945483427, 1.3340940553978193, 1.2910049686488183, 1.2527203273458163, 1.2147536525239127, 1.1769854337196524, 1.14744939519443, 1.1220955988146075, 1.0944022472959734, 1.0773022658134643, 1.0508559212107247, 1.03206209473486, 1.0208171898483585, 1.000561882500425, 0.9805161605170362, 0.9663129296065233, 0.9487424991891081, 0.931931984738623, 0.9181974345490227, 0.9075104531838204, 0.8943640941785878, 0.8862664021756573, 0.8757830774574916, 0.86790401991425, 0.8562894022841879, 0.8478155487572495, 0.8387227683262297, 0.8314011142413483, 0.8224071457687988, 0.8161875050523302, 0.8112199678685613, 0.8064334645939688 ], "peak_layer": 1, "peak_normalized": 1.3768889945483427, "population_cv": 0.16325957486062354, "rest_mean": 0.0001431148236776978, "spike_contrast": 0.8564734849544942, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001225740517838858, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0001926341064972803, 0.00018664686649572104, 0.00018061847367789596, 0.00017526224837638438, 0.0001699505082797259, 0.00016466653323732316, 0.00016053428407758474, 0.00015698715287726372, 0.00015311270544771105, 0.00015072032692842185, 0.00014702034241054207, 0.00014439098595175892, 0.00014281776384450495, 0.00013998393842484802, 0.00013717943511437625, 0.00013519232743419707, 0.00013273413060232997, 0.00013038225006312132, 0.00012846071331296116, 0.00012696554767899215, 0.0001251263020094484, 0.00012399339175317436, 0.0001225267187692225, 0.00012142439663875848, 0.00011979944974882528, 0.00011861391249112785, 0.00011734178406186402, 0.00011631744564510882, 0.00011505914153531194, 0.00011418898066040128, 0.00011349399574100971, 0.00011282433843007311 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 0.00014675117381557357, "first_to_last_ratio": 1.2676837465168667, "imbalance_abs_log_ratio": 0.23719141364257146, "last_quartile_mean": 0.00011576323686313117, "mean": 0.00013042020850662084, "normalized": [ 1.169988069421413, 1.156175152925158, 1.1417665260225645, 1.1306726331060184, 1.1183621029407895, 1.1062694829504214, 1.0940115056230622, 1.0844989956600726, 1.0711591662817939, 1.0579765491085218, 1.0443900226505625, 1.035471661013805, 1.0260498632741444, 1.013992390086173, 1.0062479316494695, 0.9977001177294099, 0.9863163473759529, 0.9769959732660659, 0.9687585669786032, 0.9602628595878113, 0.9503845997255937, 0.9426104617668664, 0.9330243108845692, 0.9259760938667744, 0.914797625470175, 0.906289421438531, 0.8956357003805738, 0.888470885240542, 0.8806216558595509, 0.8756991509238615, 0.8712858841028995, 0.8681382926882502 ], "peak_layer": 1, "peak_normalized": 1.169988069421413, "population_cv": 0.09123006364857297, "rest_mean": 0.00013202972385678786, "spike_contrast": 0.9219804606102014, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012172882561571896, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00015259008796419948, 0.00015078860451467335, 0.00014890942838974297, 0.0001474625605624169, 0.00014585701865144074, 0.00014427989663090557, 0.00014268120867200196, 0.00014144058513920754, 0.00013970080181024969, 0.0001379815221298486, 0.00013620956451632082, 0.00013504642993211746, 0.0001338176371064037, 0.00013224509893916547, 0.00013123506505507976, 0.0001301202573813498, 0.00012863558367826045, 0.00012742001854348928, 0.00012634569429792464, 0.00012523768236860633, 0.00012394935765769333, 0.00012293545296415687, 0.00012168522516731173, 0.00012076599523425102, 0.00011930809705518186, 0.00011819845531135798, 0.00011680899478960782, 0.00011587455810513347, 0.00011485085997264832, 0.00011420886585256085, 0.00011363328667357564, 0.00011322277714498341 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00030002388848515693, "first_to_last_ratio": 2.2836575149168885, "imbalance_abs_log_ratio": 0.8257783308640597, "last_quartile_mean": 0.0001313786706305109, "mean": 0.0001947244400071213, "normalized": [ 1.8998833386658438, 1.7835645331795598, 1.6713073472446418, 1.5703972875779748, 1.4780250735507643, 1.3761669788219328, 1.3066447563403325, 1.2401013786965793, 1.1762106459027857, 1.150547637187856, 1.0768735950666006, 1.0264900078881174, 1.0140784159549234, 0.9666871251601796, 0.9096841069789114, 0.8754807117769676, 0.8421876125778792, 0.8073809902734768, 0.77911667677541, 0.7607651063686129, 0.7403822030704592, 0.7308855611321243, 0.7152791523187013, 0.7043381880055872, 0.6944157299089508, 0.686864629891504, 0.680599049276497, 0.6764772714226681, 0.6702412094778135, 0.6672258960044359, 0.6635863560708457, 0.6581114274310631 ], "peak_layer": 1, "peak_normalized": 1.8998833386658438, "population_cv": 0.3592853832280775, "rest_mean": 0.00020492727344390005, "spike_contrast": 0.6813594750078023, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013962913944851608, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0003699537192005664, 0.0003473036049399525, 0.0003254443872720003, 0.00030579473241232336, 0.00028780760476365685, 0.00026797334430739284, 0.0002544356684666127, 0.0002414780465187505, 0.0002290369593538344, 0.00022403974435292184, 0.00020969360775779933, 0.00019988269195891917, 0.0001974658516701311, 0.00018823760910890996, 0.00017713772831484675, 0.00017047749133780599, 0.00016399451124016196, 0.0001572168112033978, 0.00015171305858530104, 0.00014813955931458622, 0.00014417050988413393, 0.00014232128160074353, 0.00013928233238402754, 0.0001371518592350185, 0.00013521971413865685, 0.00013374933041632175, 0.00013252926873974502, 0.00013172665785532445, 0.00013051234418526292, 0.00012992518895771354, 0.00012921648158226162, 0.000128150379168801 ] } } }, "mlp_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "afe29f3ef4d2992353f0bb59fe7c5c2c5050fca671a53184825a8d63cd83a2bc", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.0027073738165199757, 0.002722575329244137, 0.0026850115973502398, 0.0026861224323511124, 0.00271838391199708, 0.0026645169127732515, 0.0026859452482312918, 0.0027359756641089916, 0.0026767742820084095, 0.0027451682835817337, 0.0026999011170119047, 0.002663193503394723, 0.002626648638397455, 0.0026531771291047335, 0.0026517712976783514, 0.002676903735846281, 0.002670086920261383, 0.0026355870068073273, 0.0026610493659973145, 0.0026388936676084995, 0.0026469926815479994, 0.00264862016774714, 0.002641494618728757, 0.002674198243767023, 0.002557571977376938, 0.00252733426168561, 0.0025468147359788418, 0.0026856842450797558, 0.002551089273765683, 0.0025778315030038357, 0.0026001501828432083, 0.0025662274565547705 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.00015858857113926206, "first_to_last_ratio": 3.757867690304466, "imbalance_abs_log_ratio": 1.3238516928952826, "last_quartile_mean": 4.220174423608114e-05, "mean": 8.573221236929385e-05, "normalized": [ 2.5461641896458365, 2.3032502265274117, 2.0924715218207024, 1.8886722264368014, 1.7031663195050915, 1.537434754996766, 1.4223955235407997, 1.3049501874408618, 1.2055297161045848, 1.1692542365736316, 1.0769571962730058, 1.0144505847411098, 0.9913601784485884, 0.9317001386730132, 0.857757082878185, 0.8155634736467603, 0.7682612945963089, 0.7217806418748633, 0.678338533281248, 0.6511820384364709, 0.6192882306865346, 0.6053793146733001, 0.5848721334223365, 0.5718144052517429, 0.5462281883927541, 0.527515083157495, 0.5112993585545178, 0.496920907994256, 0.47722358366421574, 0.46552107689700334, 0.4601234027970099, 0.45317424906679377 ], "peak_layer": 1, "peak_normalized": 2.5461641896458365, "population_cv": 0.5686091737198813, "rest_mean": 9.231269224848668e-05, "spike_contrast": 0.5437781067692293, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.0197621021652594e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00021828828903380781, 0.00019746273756027222, 0.00017939221288543195, 0.0001619200484128669, 0.00014601621660403907, 0.000131807682919316, 0.00012194511509733275, 0.00011187626660102978, 0.00010335272963857278, 0.00010024275252362713, 9.232992306351662e-05, 8.697109296917915e-05, 8.499150135321543e-05, 7.987671415321529e-05, 7.353741239057854e-05, 6.992006092332304e-05, 6.586474046343938e-05, 6.187985127326101e-05, 5.8155463193543255e-05, 5.582727681030519e-05, 5.309295011102222e-05, 5.1900507969548926e-05, 5.0142381951445714e-05, 4.902291402686387e-05, 4.682935104938224e-05, 4.522503513726406e-05, 4.3834825191879645e-05, 4.260212881490588e-05, 4.0913433622336015e-05, 3.991015182691626e-05, 3.944739728467539e-05, 3.885163096128963e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.00021242323236947414, "first_to_last_ratio": 1.8345651540958352, "imbalance_abs_log_ratio": 0.6068074801791611, "last_quartile_mean": 0.0001157894184871111, "mean": 0.00015111673860701558, "normalized": [ 1.771707915810251, 1.6366668487510905, 1.5242909365839221, 1.4186705636795862, 1.3264449439106374, 1.2453857912414694, 1.1893415085703798, 1.1330085309323608, 1.0858463851390703, 1.062861239127963, 1.023237806768965, 0.9942126959777651, 0.978436257513136, 0.9531027445378829, 0.9232414968561398, 0.904013237249231, 0.8853704564278231, 0.8680164030054869, 0.8515402788738432, 0.8395817783841374, 0.8267435189825344, 0.8185143642831039, 0.809172703595369, 0.8007918347797284, 0.7909896391756025, 0.7821052393452608, 0.7753388667989797, 0.7682453290925442, 0.7604995795745633, 0.7549901567959042, 0.7508572350862155, 0.7467737131490535 ], "peak_layer": 1, "peak_normalized": 1.771707915810251, "population_cv": 0.2714170004285523, "rest_mean": 0.00015645500981783562, "spike_contrast": 0.7816309251520459, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012229007406858728, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0002677347220014781, 0.00024732775636948645, 0.00023034587502479553, 0.00021438486874103546, 0.00020044803386554122, 0.0001881986390799284, 0.00017972940986510366, 0.00017121655400842428, 0.00016408956435043365, 0.00016061612404882908, 0.0001546283601783216, 0.00015024218009784818, 0.00014785809617023915, 0.0001440297783119604, 0.00013951724395155907, 0.000136611532070674, 0.0001337942958343774, 0.00013117180787958205, 0.0001286819897359237, 0.00012687486014328897, 0.00012493478425312787, 0.00012369122123345733, 0.00012227953993715346, 0.00012101305037504062, 0.00011953177454415709, 0.00011818919301731512, 0.00011716668086592108, 0.00011609472858253866, 0.00011492421617731452, 0.00011409165017539635, 0.00011346709652571008, 0.00011285000800853595 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.18837904930114746, "first_to_last_ratio": 1.8345651581276443, "imbalance_abs_log_ratio": 0.6068074823768531, "last_quartile_mean": 0.10268321540206671, "mean": 0.13401183625683188, "normalized": [ 1.7717078885231814, 1.6366668119471057, 1.524290957183596, 1.418670514273678, 1.326444920903592, 1.2453858718827995, 1.1893415500975306, 1.1330085716119305, 1.0858463365841577, 1.062861212873729, 1.0232377434953237, 0.9942127340740732, 0.9784362443707608, 0.9531027271661117, 0.9232414917376033, 0.904013241653558, 0.885370477646398, 0.8680164390463408, 0.8515402681847718, 0.8395816969955481, 0.8267435345782925, 0.8185144279720658, 0.8091727254708025, 0.8007918405918669, 0.7909896328038464, 0.7821052665924377, 0.7753388466443417, 0.7682453528580236, 0.7604996016741158, 0.7549901615676585, 0.7508572336421532, 0.7467736753526057 ], "peak_layer": 1, "peak_normalized": 1.7717078885231814, "population_cv": 0.27141699472708714, "rest_mean": 0.1387458680956452, "spike_contrast": 0.7816309473985974, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.10844806432724, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.23742982745170593, 0.2193327248096466, 0.2042730301618576, 0.19011864066123962, 0.1777593195438385, 0.16689644753932953, 0.15938584506511688, 0.151836559176445, 0.1455162614583969, 0.1424359828233719, 0.13712596893310547, 0.13323627412319183, 0.13112203776836395, 0.12772704660892487, 0.1237252876162529, 0.1211484745144844, 0.11865012347698212, 0.11632447689771652, 0.11411647498607635, 0.11251388490200043, 0.11079341918230057, 0.10969062149524689, 0.10843872278928757, 0.10731558501720428, 0.10600197315216064, 0.10481136292219162, 0.10390458256006241, 0.1029539704322815, 0.10191594809293747, 0.10117761790752411, 0.10062375664710999, 0.10007651150226593 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 6.873187476230669e-05, "first_to_last_ratio": 2.248757500145603, "imbalance_abs_log_ratio": 0.8103778415280068, "last_quartile_mean": 3.0564378221242805e-05, "mean": 4.685761763312257e-05, "normalized": [ 1.7400006587426948, 1.6405194725375354, 1.5551306973978087, 1.4944078341288562, 1.4327724019957897, 1.3414187773842097, 1.2804088050749736, 1.2499327782691996, 1.1927955171655855, 1.1608931065609767, 1.0981756972550076, 1.0577423904261343, 1.0542786038595535, 1.0218640087393278, 0.9775807389332813, 0.9570972402588515, 0.9209982861589985, 0.8815092249082567, 0.8478232134430647, 0.8204952153707635, 0.7871780668180614, 0.7759097745574741, 0.7506963524238334, 0.7421155343193898, 0.7072927325438048, 0.6891814936654451, 0.6695399899996352, 0.6616343983211914, 0.6345111343049281, 0.6212687927046295, 0.6200231913288072, 0.6148038704019305 ], "peak_layer": 1, "peak_normalized": 1.7400006587426948, "population_cv": 0.3217536161653951, "rest_mean": 4.900405669845727e-05, "spike_contrast": 0.719671983430414, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.526684668031521e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 8.153228554874659e-05, 7.687083416385576e-05, 7.286971958819777e-05, 7.002439087955281e-05, 6.71363013680093e-05, 6.285568815656006e-05, 5.999690620228648e-05, 5.8568872191244736e-05, 5.5891556257847697e-05, 5.439668530016206e-05, 5.145789691596292e-05, 4.9563288484932855e-05, 4.9400983698433265e-05, 4.7882112994557247e-05, 4.580710447044112e-05, 4.484729652176611e-05, 4.3155785533599555e-05, 4.130542220082134e-05, 3.972697595600039e-05, 3.844645107164979e-05, 3.688528886414133e-05, 3.635728353401646e-05, 3.517584264045581e-05, 3.477376594673842e-05, 3.314205241622403e-05, 3.229340290999971e-05, 3.1373048841487616e-05, 3.10026116494555e-05, 2.9731680115219206e-05, 2.9111175535945222e-05, 2.9052809622953646e-05, 2.8808244678657502e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 0.0001719075116852764, "first_to_last_ratio": 1.4894635014227988, "imbalance_abs_log_ratio": 0.39841598895878805, "last_quartile_mean": 0.00011541572621354135, "mean": 0.00013894635094402474, "normalized": [ 1.3738985779819257, 1.3296082759737424, 1.2883384004275333, 1.2499940655466777, 1.2130730176151365, 1.1755503979222366, 1.147409009170091, 1.11990595257071, 1.0929037170303153, 1.075598370036652, 1.0512762916941902, 1.0312594730951, 1.0194418974669357, 1.0018996501730997, 0.9803460168062212, 0.9648004557293595, 0.9493010807852901, 0.9344788929945884, 0.9198953859218769, 0.908001147597105, 0.8952913731815082, 0.886953676542665, 0.8771894463102521, 0.8683888934041172, 0.8579399877904575, 0.8482799643655919, 0.8408636339472488, 0.8330870307470335, 0.82434015220157, 0.8181187963300758, 0.8135579418994486, 0.809009026741244 ], "peak_layer": 1, "peak_normalized": 1.3738985779819257, "population_cv": 0.16161423472023242, "rest_mean": 0.0001421073115866252, "spike_contrast": 0.8576417505420824, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012187716347398237, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0001908981939777732, 0.0001847442181315273, 0.00017900991952046752, 0.00017368211410939693, 0.00016855206922627985, 0.00016333843814209104, 0.00015942829486448318, 0.00015560684551019222, 0.0001518549834145233, 0.00014945046859793365, 0.00014607100456487387, 0.0001432897406630218, 0.00014164773165248334, 0.00013921030040364712, 0.000136215501697734, 0.0001340555027127266, 0.0001319019211223349, 0.00012984243221580982, 0.0001278161071240902, 0.00012616344611160457, 0.00012439746933523566, 0.00012323897681199014, 0.00012188227265141904, 0.00012065946793882176, 0.00011920763063244522, 0.00011786540562752634, 0.00011683493357850239, 0.00011575440294109285, 0.0001145390560850501, 0.00011367462138878182, 0.00011304090730845928, 0.00011240885214647278 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 0.00014552221000485588, "first_to_last_ratio": 1.262697055715921, "imbalance_abs_log_ratio": 0.2332499537229619, "last_quartile_mean": 0.00011524712863320019, "mean": 0.00012953460168318998, "normalized": [ 1.1676213631634658, 1.1525055688027706, 1.1391580058081985, 1.1289307982409096, 1.1166731577821778, 1.1054001772932092, 1.094813144753513, 1.0822854389705017, 1.0679308603491235, 1.0564894822224282, 1.0456838360716472, 1.0337072702557388, 1.0234945545465333, 1.0158819477934218, 1.0051772952892828, 0.9951467950359763, 0.9866643397779161, 0.9787559418439278, 0.970448736962688, 0.9607908684400951, 0.9509939230158972, 0.9428061355388626, 0.934924980051271, 0.9261034265480621, 0.9170582603044295, 0.9058562786860701, 0.8981778407016658, 0.8907564927732574, 0.8831168120837269, 0.8778216107141017, 0.8745192645108896, 0.8703053916682415 ], "peak_layer": 1, "peak_normalized": 1.1676213631634658, "population_cv": 0.08967236486753827, "rest_mean": 0.00013110875065817877, "spike_contrast": 0.9231588022206524, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012103419721825049, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00015124736819416285, 0.00014928934979252517, 0.000147560378536582, 0.00014623560127802193, 0.00014464781270362437, 0.00014318757166620344, 0.00014181618462316692, 0.00014019341324456036, 0.0001383339986205101, 0.00013685194426216185, 0.00013545223919209093, 0.00013390085950959474, 0.00013257795944809914, 0.00013159186346456409, 0.00013020524056628346, 0.00012890594371128827, 0.00012780717224813998, 0.00012678276107180864, 0.0001257066905964166, 0.0001244556624442339, 0.00012318661902099848, 0.0001221260172314942, 0.00012110513489460573, 0.0001199624384753406, 0.00011879077646881342, 0.00011733973224181682, 0.00011634510883595794, 0.00011538378748809919, 0.0001143941844929941, 0.00011370827269274741, 0.00011328050459269434, 0.0001127346622524783 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00029741139769612346, "first_to_last_ratio": 2.278072415841521, "imbalance_abs_log_ratio": 0.8233296537390045, "last_quartile_mean": 0.00013055396993877366, "mean": 0.00019328556300024502, "normalized": [ 1.889398576482247, 1.780233894901915, 1.6772594716902607, 1.566224958959574, 1.4762232610465842, 1.375571144634086, 1.3083686481730985, 1.2364401527267181, 1.173148897934763, 1.1508864184295666, 1.080457040307468, 1.0318582947152708, 1.0081957004611959, 0.9698994715426978, 0.9066127339785177, 0.8754550048258108, 0.8446146862114313, 0.8115724449181009, 0.7806677558079268, 0.7613480996239201, 0.7403134026279838, 0.7294052762839086, 0.7160186976339193, 0.7062575707090604, 0.6942777320753531, 0.6866005528973866, 0.6829938476675228, 0.6753403086482458, 0.6715967314167978, 0.6686860556999406, 0.6640286432895989, 0.6600445236791291 ], "peak_layer": 1, "peak_normalized": 1.889398576482247, "population_cv": 0.3579154260008552, "rest_mean": 0.00020340604748559632, "spike_contrast": 0.681567477924495, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013863494677934796, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0003651934675872326, 0.0003440935106482357, 0.00032419004128314555, 0.0003027286729775369, 0.0002853326441254467, 0.00026587804313749075, 0.00025288877077400684, 0.00023898603103589267, 0.00022675274522043765, 0.00022244972933549434, 0.0002088367473334074, 0.00019944331143051386, 0.00019486967357806861, 0.00018746756541077048, 0.00017523515271022916, 0.00016921281348913908, 0.0001632518251426518, 0.00015686523693148047, 0.00015089180669747293, 0.00014715759607497603, 0.00014309189282357693, 0.00014098350948188454, 0.00013839607709087431, 0.0001365093921776861, 0.0001341938623227179, 0.00013270997442305088, 0.00013201285037212074, 0.00013053353177383542, 0.0001298099523410201, 0.00012924736074637622, 0.00012834715016651899, 0.00012757707736454904 ] } } }, "post_attention_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "31853ffb07b24cc59e774bb20656840b4358b3080b42516d380f2dc0f46404c8", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.0022265110164880753, 0.0040087271481752396, 0.005379301495850086, 0.006410556845366955, 0.0027253651060163975, 0.004838948138058186, 0.006175771821290255, 0.00742313964292407, 0.0030937574338167906, 0.005150026176124811, 0.007019025273621082, 0.008143599145114422, 0.004255065228790045, 0.0066224560141563416, 0.008129785768687725, 0.010099578648805618, 0.0041999719105660915, 0.00719039561226964, 0.009030314162373543, 0.010736005380749702, 0.005131533369421959, 0.00787449348717928, 0.009912566281855106, 0.012113703414797783, 0.0059690922498703, 0.00876442901790142, 0.010806876234710217, 0.012353328987956047, 0.006502739619463682, 0.009470963850617409, 0.012123330496251583, 0.01384514570236206 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.00015962236011546338, "first_to_last_ratio": 3.7698422272252903, "imbalance_abs_log_ratio": 1.3270331510418396, "last_quartile_mean": 4.2341920561739244e-05, "mean": 8.61727822893954e-05, "normalized": [ 2.542734798705971, 2.3221448837042376, 2.101145461274622, 1.8911858524935854, 1.7004204056817789, 1.5354142426015858, 1.4229001770142031, 1.3028733810328659, 1.2049412416934622, 1.1705258152816296, 1.0773908232649334, 1.0133220774376726, 0.990609360100461, 0.9272534949358001, 0.8550385909364457, 0.8185651575045715, 0.7703972051466339, 0.7173947009730917, 0.6760256695667757, 0.6514523109028573, 0.618419077603279, 0.6053620388084041, 0.5838930841859182, 0.5697043695032278, 0.5457802700901755, 0.5276149005446632, 0.5104701332743111, 0.49517988582787786, 0.47527562953524777, 0.46579059316028215, 0.45896435764104826, 0.45181000957238127 ], "peak_layer": 1, "peak_normalized": 2.542734798705971, "population_cv": 0.5705202801130907, "rest_mean": 9.280119640472414e-05, "spike_contrast": 0.542873885449774, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.03793460666202e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00021911453222855926, 0.00020010568550787866, 0.00018106155039276928, 0.00016296874673571438, 0.00014652995741926134, 0.00013231091725174338, 0.0001226152671733871, 0.00011227222421439365, 0.00010383313929196447, 0.00010086746624438092, 9.284176485380158e-05, 8.732078276807442e-05, 8.536356472177431e-05, 7.99040135461837e-05, 7.368105434579775e-05, 7.053803710732609e-05, 6.638727063545957e-05, 6.181989738252014e-05, 5.825501284562051e-05, 5.613745815935545e-05, 5.329089253791608e-05, 5.216573117650114e-05, 5.0315691623836756e-05, 4.9093010602518916e-05, 4.703140439232811e-05, 4.546604395727627e-05, 4.398863165988587e-05, 4.267102849553339e-05, 4.095582335139625e-05, 4.013847137684934e-05, 3.955023566959426e-05, 3.893372559105046e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.00021442477373057045, "first_to_last_ratio": 1.8431254562763404, "imbalance_abs_log_ratio": 0.6114627481432037, "last_quartile_mean": 0.00011633759004325839, "mean": 0.00015221248349917005, "normalized": [ 1.7740805953138137, 1.6484457246979003, 1.5301434809496548, 1.4217241600875643, 1.3264958046529274, 1.2454577490392524, 1.1897861330408546, 1.1336265611690741, 1.086914920172498, 1.064301358569379, 1.0227209980571423, 0.9941753882589759, 0.9788979908211966, 0.9511107967643203, 0.9228143274453928, 0.9052447611165039, 0.8846530997418666, 0.8652930871753776, 0.8497381552743479, 0.8388656494639406, 0.8256334792987466, 0.8176309647342147, 0.8076951735882674, 0.8000661307359944, 0.7892605467734257, 0.78144613040264, 0.7731219607566042, 0.7664988488799842, 0.7584935618388989, 0.7529639051546407, 0.7485195296370439, 0.7441790263875565 ], "peak_layer": 1, "peak_normalized": 1.7740805953138137, "population_cv": 0.2736411578809574, "rest_mean": 0.00015762286892384981, "spike_contrast": 0.7803207938393821, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012299640220589936, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00027003721334040165, 0.00025091401766985655, 0.00023290693934541196, 0.0002164041652576998, 0.000201909220777452, 0.0001895742170745507, 0.00018110030214302242, 0.00017255211423616856, 0.00016544201935175806, 0.00016199995297938585, 0.00015567090304102749, 0.00015132590488065034, 0.0001490004942752421, 0.00014477093645837158, 0.00014046386058907956, 0.00013778955326415598, 0.0001346552453469485, 0.00013170840975362808, 0.00012934075493831187, 0.00012768582382705063, 0.00012567172234412283, 0.00012445363972801715, 0.00012294128828216344, 0.00012178005272289738, 0.00012013530795229599, 0.00011894585622940212, 0.00011767881369451061, 0.00011667069338727742, 0.00011545218876563013, 0.00011461050598882139, 0.00011393401655368507, 0.00011327333777444437 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.190154030919075, "first_to_last_ratio": 1.84312545379361, "imbalance_abs_log_ratio": 0.6114627467961817, "last_quartile_mean": 0.10316933691501617, "mean": 0.13498355145566165, "normalized": [ 1.774080628234443, 1.6484456620184311, 1.5301434020504485, 1.4217242444591272, 1.326495766996536, 1.2454576834375484, 1.189786122599316, 1.133626627255238, 1.0869149569673349, 1.0643014004698326, 1.0227209907477024, 0.9941753870542451, 0.9788979589698585, 0.9511107512164827, 0.9228143032197887, 0.9052447972745712, 0.8846531331847195, 0.8652931173486506, 0.8497382179112226, 0.8388656682549475, 0.8256334805754935, 0.8176309131099365, 0.8076951535479161, 0.8000661540388709, 0.7892605573866218, 0.7814461538045975, 0.7731219580171443, 0.7664988542586116, 0.7584935821787124, 0.7529638601652359, 0.7485195164462104, 0.7441789968002049 ], "peak_layer": 1, "peak_normalized": 1.774080628234443, "population_cv": 0.27364115345546325, "rest_mean": 0.13978153549962574, "spike_contrast": 0.7803207857775153, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.10907443761825561, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.2394717037677765, 0.22251304984092712, 0.2065441906452179, 0.19190938770771027, 0.1790551096200943, 0.16811630129814148, 0.16060155630111694, 0.1530209481716156, 0.14671564102172852, 0.1436631828546524, 0.13805051147937775, 0.13419732451438904, 0.13213512301445007, 0.1283843070268631, 0.1245647519826889, 0.12219315767288208, 0.11941362172365189, 0.11680033802986145, 0.11470068246126175, 0.1132330670952797, 0.11144693940877914, 0.11036672443151474, 0.10902556031942368, 0.10799577087163925, 0.10653719305992126, 0.10548237711191177, 0.1043587476015091, 0.10346473753452301, 0.10238415747880936, 0.10163773596286774, 0.10103782266378403, 0.10045192390680313 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 6.883367768750759e-05, "first_to_last_ratio": 2.249759357551583, "imbalance_abs_log_ratio": 0.810823258297238, "last_quartile_mean": 3.059601795030176e-05, "mean": 4.691846504556452e-05, "normalized": [ 1.7359059710084281, 1.6475789727938472, 1.5536456441204052, 1.4921194549590373, 1.4365657321885223, 1.3438513058553838, 1.2787343947807137, 1.2483299149152907, 1.1912354282513808, 1.1603083401306369, 1.099547144002948, 1.059972440022979, 1.0583955432739494, 1.0176800955770247, 0.9754963047746087, 0.9631595708780456, 0.9242400656751482, 0.8752301116137658, 0.8454245377074081, 0.8232853322610043, 0.7863494898171388, 0.7774642190168813, 0.7497486129019063, 0.7388483538626245, 0.709315327139865, 0.6908343794807076, 0.6685469983108738, 0.6592365073812129, 0.6314341746588822, 0.6235898169926835, 0.6199474545603838, 0.6139783610863121 ], "peak_layer": 1, "peak_normalized": 1.7359059710084281, "population_cv": 0.3220406003070692, "rest_mean": 4.9070239684725595e-05, "spike_contrast": 0.7193541792517962, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.52988819940947e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 8.144604362314567e-05, 7.730187644483522e-05, 7.28946688468568e-05, 7.000795449130237e-05, 6.740145909134299e-05, 6.305144052021205e-05, 5.999625500408001e-05, 5.8569723478285596e-05, 5.589093780145049e-05, 5.4439886298496276e-05, 5.158906424185261e-05, 4.973227987647988e-05, 4.9658294301480055e-05, 4.774798799189739e-05, 4.576878927764483e-05, 4.51899686595425e-05, 4.3363925215089694e-05, 4.1064453398576006e-05, 3.9666021621087566e-05, 3.86272840842139e-05, 3.689431105158292e-05, 3.6477427784120664e-05, 3.5177054087398574e-05, 3.466563066467643e-05, 3.3279986382694915e-05, 3.2412888685939834e-05, 3.136719897156581e-05, 3.093036502832547e-05, 2.9625922252307646e-05, 2.9257877031341195e-05, 2.9086982976878062e-05, 2.8806922273361124e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 0.00017341253987979144, "first_to_last_ratio": 1.4955148931299893, "imbalance_abs_log_ratio": 0.4024705576674927, "last_quartile_mean": 0.00011595507385209203, "mean": 0.00013990553861731314, "normalized": [ 1.3768818258178377, 1.3340879911950096, 1.2910054180628048, 1.2527236427111486, 1.2147542166907819, 1.1769889670731195, 1.1474411155803916, 1.1220953663134672, 1.0944075691302455, 1.0773007485615413, 1.0508678613623372, 1.0320596054419082, 1.0208161736522385, 1.000559648637946, 0.9805197815181176, 0.9663128275349431, 0.948744191771126, 0.9319305822463463, 0.9182019814348666, 0.9075038868226692, 0.8943650364968407, 0.886262676120959, 0.8757907045344768, 0.8679002174812178, 0.856288373659771, 0.8478178092769747, 0.8387215581135022, 0.8313975747674917, 0.8224084044020752, 0.8161864847722404, 0.8112200992010112, 0.8064376596145924 ], "peak_layer": 1, "peak_normalized": 1.3768818258178377, "population_cv": 0.16325888721442564, "rest_mean": 0.0001431150389921472, "spike_contrast": 0.8564734877369171, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001225742365932092, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0001926333934534341, 0.00018664629897102714, 0.00018061880837194622, 0.0001752629759721458, 0.00016995084297377616, 0.0001646672753849998, 0.00016053336730692536, 0.00015698735660407692, 0.00015311368042603135, 0.00015072034148033708, 0.00014702223415952176, 0.00014439085498452187, 0.0001428178366040811, 0.00013998383656144142, 0.00013718014815822244, 0.00013519251660909504, 0.00013273456715978682, 0.00013038225006312132, 0.00012846154277212918, 0.00012696482008323073, 0.00012512662215158343, 0.0001239930570591241, 0.00012252797023393214, 0.000121424047392793, 0.00011979948612861335, 0.00011861440725624561, 0.00011734179133782163, 0.0001163171255029738, 0.00011505949078127742, 0.00011418900976423174, 0.00011349418491590768, 0.00011282509512966499 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 0.00014675183047074825, "first_to_last_ratio": 1.2676545313577883, "imbalance_abs_log_ratio": 0.23716836728277432, "last_quartile_mean": 0.00011576642282307148, "mean": 0.00013041823922321782, "normalized": [ 1.1700698937843759, 1.1560440994564782, 1.1417398043557874, 1.130463200949706, 1.118403537277814, 1.1065304334184665, 1.0939881912841023, 1.0846815122144253, 1.0709468270677047, 1.0579907390333845, 1.0442838370166418, 1.035550673160169, 1.025959021745254, 1.0140203095307772, 1.0062047700154308, 0.9976631870190885, 0.9861765922450596, 0.9770016877798172, 0.9687702939454803, 0.9602422119882037, 0.9504995944091904, 0.9425883202531964, 0.9330014667281973, 0.9259385264566372, 0.9147443798098084, 0.906202796813466, 0.8958065504175365, 0.8885655303249116, 0.8806194435723184, 0.8757905905296257, 0.8713783728577705, 0.8681336045391749 ], "peak_layer": 1, "peak_normalized": 1.1700698937843759, "population_cv": 0.09121429133712369, "rest_mean": 0.0001320278307604086, "spike_contrast": 0.921975648780332, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001217264449223876, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0001525984553154558, 0.0001507692359155044, 0.000148903694935143, 0.00014743302017450333, 0.00014586022007279098, 0.00014431175077334046, 0.00014267601363826543, 0.00014146225294098258, 0.000139670999487862, 0.00013798128929920495, 0.00013619365927297622, 0.00013505469541996717, 0.00013380376913119107, 0.00013224674330558628, 0.00013122745440341532, 0.00013011347618885338, 0.00012861541472375393, 0.00012741883983835578, 0.0001263453159481287, 0.0001252330985153094, 0.0001239624834852293, 0.00012293070903979242, 0.00012168040848337114, 0.00012075927224941552, 0.00011929935135412961, 0.00011818537313956767, 0.00011682951299007982, 0.00011588515189941972, 0.00011484883725643158, 0.00011421906674513593, 0.00011364363308530301, 0.00011322045611450449 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00030004180916876066, "first_to_last_ratio": 2.2837506327398005, "imbalance_abs_log_ratio": 0.8258191057718037, "last_quartile_mean": 0.00013138116082700435, "mean": 0.00019473525253488333, "normalized": [ 1.9000096511818072, 1.783700741635724, 1.6708066910504589, 1.570366884717585, 1.4783058799917985, 1.376185321596543, 1.3064973297630273, 1.2402700043336092, 1.1767953849206394, 1.1505341943958838, 1.0771542571420398, 1.026098162637436, 1.013898063759585, 0.9669473025647412, 0.9095398902231254, 0.8755205777387188, 0.842301214161826, 0.8073124727024382, 0.7791991818813049, 0.760574607765185, 0.7402055397447304, 0.7308143413325056, 0.7152667122155245, 0.7043714155678553, 0.6942467749765324, 0.6868749151283189, 0.680655564574882, 0.676363713518914, 0.6701061029149729, 0.6672276319278244, 0.6636177363814285, 0.6582317375530352 ], "peak_layer": 1, "peak_normalized": 1.9000096511818072, "population_cv": 0.35930746026536914, "rest_mean": 0.00020494150898971216, "spike_contrast": 0.6812747128050888, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013962146767880766, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0003699988592416048, 0.0003473494143690914, 0.0003253649629186839, 0.0003058057918678969, 0.0002878782688640058, 0.0002679917961359024, 0.0002544210874475539, 0.00024152429250534624, 0.00022916354646440595, 0.000224049566895701, 0.00020975990628357977, 0.00019981748482678086, 0.000197441695490852, 0.0001882987271528691, 0.00017711948021315038, 0.00017049472080543637, 0.00016402573965024203, 0.00015721219824627042, 0.00015173754945863038, 0.00014811068831477314, 0.00014414411270990968, 0.0001423153153154999, 0.0001392876438330859, 0.00013716594548895955, 0.00013519432104658335, 0.00013375876005738974, 0.00013254763325676322, 0.0001317118585575372, 0.00013049328117631376, 0.00012993274140171707, 0.00012922976748086512, 0.00012818092363886535 ] } } }, "post_mlp_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "7fd9c911e93175896ad618842c120e77b3f017c17b7eec9c2bfb01fc6dbc8ac2", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.0034885562490671873, 0.004831995815038681, 0.005952076520770788, 0.006925336085259914, 0.00385080324485898, 0.005542940925806761, 0.006729288958013058, 0.007927131839096546, 0.004081308841705322, 0.005811207927763462, 0.007534808944910765, 0.008581922389566898, 0.00499920267611742, 0.007102519739419222, 0.008542952127754688, 0.010447698645293713, 0.005041283555328846, 0.007636935915797949, 0.009387043304741383, 0.011020516976714134, 0.005807018838822842, 0.008367395959794521, 0.01024632342159748, 0.012378672137856483, 0.006453148555010557, 0.009127611294388771, 0.011037957854568958, 0.012611051090061665, 0.007010402157902718, 0.009794782847166061, 0.012445803731679916, 0.014087481424212456 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.00015858902861509705, "first_to_last_ratio": 3.757869257587208, "imbalance_abs_log_ratio": 1.3238521099622325, "last_quartile_mean": 4.220184837322449e-05, "mean": 8.57325238712292e-05, "normalized": [ 2.5462257183789148, 2.3032637538343064, 2.0924289533379263, 1.8886492391629932, 1.7031256747388024, 1.5374176267930122, 1.4224231993460965, 1.3049597038569594, 1.2055618292025094, 1.169245405308266, 1.0769812048488787, 1.0144334047898542, 0.9913512297333369, 0.9317090592952362, 0.8577499774884012, 0.81555804918997, 0.7682725064221458, 0.7217807351250844, 0.6783324192661732, 0.6511800118936032, 0.6192916242842709, 0.6053728292385717, 0.5848720876072855, 0.5718124973482859, 0.5462302773879167, 0.5275173250132701, 0.5112998770983732, 0.49691833866078555, 0.47721658788927634, 0.46551989467389665, 0.46012635628943105, 0.45317260249616614 ], "peak_layer": 1, "peak_normalized": 2.5462257183789148, "population_cv": 0.5686111305901927, "rest_mean": 9.231303704803361e-05, "spike_contrast": 0.5437775023084314, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.019775271648541e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00021829435718245804, 0.00019746461475733668, 0.0001793892151908949, 0.0001619186659809202, 0.00014601326256524771, 0.00013180669338908046, 0.00012194793089292943, 0.00011187748896190897, 0.00010335585830034688, 0.00010024235962191597, 9.233231685357168e-05, 8.69699360919185e-05, 8.499104296788573e-05, 7.987776916706935e-05, 7.353707042057067e-05, 6.991984992055222e-05, 6.586594099644572e-05, 6.188008410390466e-05, 5.8155150327365845e-05, 5.582730591413565e-05, 5.309343396220356e-05, 5.190014053368941e-05, 5.014256021240726e-05, 4.90229285787791e-05, 4.682970029534772e-05, 4.522539165918715e-05, 4.383502891869284e-05, 4.2602063331287354e-05, 4.0912982512963936e-05, 3.991019548266195e-05, 3.944779382436536e-05, 3.885163096128963e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.00021242368711682502, "first_to_last_ratio": 1.8345658968465297, "imbalance_abs_log_ratio": 0.6068078850437921, "last_quartile_mean": 0.0001157896194854402, "mean": 0.0001511171253696375, "normalized": [ 1.7717451736709768, 1.636674793183293, 1.5242755762025908, 1.4186515254985965, 1.3264266232459345, 1.2453738409548603, 1.1893512719369177, 1.1330135273991528, 1.085860457802012, 1.0628542818802846, 1.023252039669589, 0.9942070699734702, 0.9784308644775869, 0.9531083940389613, 0.9232390376562555, 0.9040072643238835, 0.8853775311228679, 0.8680181295617843, 0.8515367513400655, 0.8395795332993111, 0.8267439067367663, 0.8185133286645813, 0.8091730400239778, 0.800791325997512, 0.7909899739933552, 0.7821072339266707, 0.7753403490721149, 0.7682422073314642, 0.7604971517068638, 0.7549863467424895, 0.7508596466770359, 0.7467718018887739 ], "peak_layer": 1, "peak_normalized": 1.7717451736709768, "population_cv": 0.2714186609524326, "rest_mean": 0.00015645541295978345, "spike_contrast": 0.7816308178118705, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012229037238284946, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0002677410375326872, 0.0002473295899108052, 0.00023034414334688336, 0.00021438254043459892, 0.00020044577831868082, 0.00018819731485564262, 0.00017973134526982903, 0.000171217747265473, 0.0001640921109355986, 0.00016061548376455903, 0.00015463090676348656, 0.00015024171443656087, 0.0001478576596127823, 0.00014403100067283958, 0.00013951722939964384, 0.00013661097909789532, 0.00013379570737015456, 0.0001311724045081064, 0.0001286817860091105, 0.00012687484559137374, 0.0001249351626029238, 0.00012369138130452484, 0.00012227990373503417, 0.00012101328320568427, 0.00011953213106608018, 0.0001181897969217971, 0.0001171672047348693, 0.00011609455395955592, 0.00011492414341773838, 0.0001140913664130494, 0.00011346775136189535, 0.00011285000800853595 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.18837944976985455, "first_to_last_ratio": 1.8345658800697515, "imbalance_abs_log_ratio": 0.6068078758989703, "last_quartile_mean": 0.10268339328467846, "mean": 0.1340121782850474, "normalized": [ 1.7717451751532867, 1.6366747548098712, 1.5242756140248972, 1.4186514377480737, 1.3264266357207033, 1.2453739091726768, 1.1893513017871604, 1.1330134634090196, 1.0858604665419038, 1.0628542749024097, 1.0232520332438115, 0.9942070832344971, 0.9784308561813967, 0.9531084116998004, 0.9232390798282929, 0.9040072094634526, 0.8853775581660035, 0.8680181154203164, 0.8515367049614931, 0.8395794986042229, 0.8267439263823189, 0.8185133396785964, 0.809173050926764, 0.8007913534932437, 0.7909900046670136, 0.782107217826699, 0.7753403703776178, 0.7682422246066585, 0.7604971603471317, 0.7549863443945886, 0.7508596538022533, 0.7467717694238248 ], "peak_layer": 1, "peak_normalized": 1.7717451751532867, "population_cv": 0.27141865758333467, "rest_mean": 0.13874622461972413, "spike_contrast": 0.7816308398662988, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.10844832807779312, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.23743543028831482, 0.21933434903621674, 0.20427149534225464, 0.19011656939983368, 0.17775732278823853, 0.16689527034759521, 0.15938755869865417, 0.15183760225772858, 0.14551852643489838, 0.14243541657924652, 0.13712823390960693, 0.1332358568906784, 0.1311216503381729, 0.12772813439369202, 0.1237252801656723, 0.12114797532558441, 0.1186513751745224, 0.1163249984383583, 0.11411628872156143, 0.11251387745141983, 0.11079375445842743, 0.10969075560569763, 0.10843904316425323, 0.107315793633461, 0.10600229352712631, 0.104811891913414, 0.10390505194664001, 0.10295381397008896, 0.1019158810377121, 0.10117736458778381, 0.10062433779239655, 0.10007651150226593 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 6.873207303215167e-05, "first_to_last_ratio": 2.248757814140914, "imbalance_abs_log_ratio": 0.8103779811585758, "last_quartile_mean": 3.0564462122129044e-05, "mean": 4.6857780716891284e-05, "normalized": [ 1.7400119939228862, 1.6405409364325143, 1.555121247708426, 1.4944054279879166, 1.4327475398601994, 1.3414100714975594, 1.2804101716466747, 1.2499370459040815, 1.192804641985012, 1.1608858830017554, 1.098191983597808, 1.057744066133373, 1.0542717513613309, 1.021869458338179, 0.9775749297615711, 0.9570927445976433, 0.9210091333285576, 0.8815125232807631, 0.847820728512489, 0.8204970180441978, 0.7871838673851995, 0.7759041238100245, 0.7506938949785397, 0.7421070509174572, 0.7072987335050381, 0.6891811912847667, 0.6695407652827761, 0.6616333377893114, 0.6345027924930273, 0.6212629426015038, 0.6200302724102036, 0.614801730639213 ], "peak_layer": 1, "peak_normalized": 1.7400119939228862, "population_cv": 0.32175451551289735, "rest_mean": 4.9004235565748616e-05, "spike_contrast": 0.7196709452950106, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.5266924533061685e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 8.153310045599937e-05, 7.687210745643824e-05, 7.28695304132998e-05, 7.002452184678987e-05, 6.713537004543468e-05, 6.28554989816621e-05, 5.9997179050697014e-05, 5.8569276006892323e-05, 5.589217835222371e-05, 5.439653614303097e-05, 5.1458839152473956e-05, 4.9563539505470544e-05, 4.9400834541302174e-05, 4.788253500009887e-05, 4.58069916930981e-05, 4.4847241952084005e-05, 4.315644400776364e-05, 4.130572051508352e-05, 3.972699778387323e-05, 3.8446669350378215e-05, 3.688568904181011e-05, 3.635714529082179e-05, 3.5175849916413426e-05, 3.477348946034908e-05, 3.3142448955914006e-05, 3.2293501135427505e-05, 3.13731943606399e-05, 3.1002669857116416e-05, 2.9731392714893445e-05, 2.9111002731951885e-05, 2.905324254243169e-05, 2.8808244678657502e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 0.00017190765356644988, "first_to_last_ratio": 1.4894621368061003, "imbalance_abs_log_ratio": 0.3984150727783497, "last_quartile_mean": 0.00011541592721187044, "mean": 0.00013894661265112518, "normalized": [ 1.3739040544627474, 1.3296102750467884, 1.2883342981472405, 1.2499918159027226, 1.2130627732820631, 1.175544413473833, 1.1474083142393565, 1.1199112790671786, 1.092909618037032, 1.0755929927697745, 1.0512861461262244, 1.0312571117882559, 1.0194388253067554, 1.0019061415077835, 0.980344379776871, 0.9647961249933756, 0.9493075664595542, 0.9344817410273759, 0.9198929201794295, 0.9079992279088673, 0.8952920956888009, 0.8869530532606651, 0.8771902552778059, 0.8683889858357474, 0.8579407806493952, 0.8482823463732029, 0.8408658204630546, 0.8330843095877946, 0.8243382853592501, 0.818115422616924, 0.8135611224181157, 0.8090075029660142 ], "peak_layer": 1, "peak_normalized": 1.3739040544627474, "population_cv": 0.16161403917150835, "rest_mean": 0.00014210757001786045, "spike_contrast": 0.8576421569620596, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012187744287075474, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00019089931447524577, 0.00018474484386388212, 0.00017900968668982387, 0.00017368212866131216, 0.0001685509632807225, 0.00016333791427314281, 0.00015942849859129637, 0.00015560787869617343, 0.00015185608936008066, 0.00014945000293664634, 0.00014607264893129468, 0.0001432896824553609, 0.00014164757158141583, 0.0001392114645568654, 0.00013621553080156446, 0.0001340551534667611, 0.00013190307072363794, 0.00012984307250007987, 0.0001278160052606836, 0.00012616341700777411, 0.0001243978040292859, 0.00012323912233114243, 0.00012188261462142691, 0.00012065970804542303, 0.00011920796532649547, 0.00011786595860030502, 0.00011683545744745061, 0.00011575424287002534, 0.00011453901242930442, 0.00011367436673026532, 0.00011304156214464456, 0.00011240885214647278 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 0.00014552358334185556, "first_to_last_ratio": 1.262638843036328, "imbalance_abs_log_ratio": 0.2332038508028458, "last_quartile_mean": 0.00011525352965691127, "mean": 0.00012953498708156985, "normalized": [ 1.1676304712442283, 1.1525836984023852, 1.1391521450612143, 1.1288261090295841, 1.1167600441451297, 1.1054423860149858, 1.094731137320412, 1.082319740349084, 1.067957453000188, 1.0563935463552665, 1.04560972624621, 1.0337959762176818, 1.0235535208863231, 1.0159040893770892, 1.0050628637064216, 0.9950792389307415, 0.9865814183778941, 0.9786230528286792, 0.9704418054120744, 0.9608381133357455, 0.9508407831226554, 0.9429749854457838, 0.9346839821902475, 0.9262276149685831, 0.9170542960948607, 0.9059488475652709, 0.8984648923618161, 0.8908037775288885, 0.8831483920181569, 0.8778579808382188, 0.8744051093280881, 0.8703028022960919 ], "peak_layer": 1, "peak_normalized": 1.1676304712442283, "population_cv": 0.0896642712448568, "rest_mean": 0.00013110964479252557, "spike_contrast": 0.9231344927669946, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012103183544240892, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00015124899800866842, 0.00014929991448298097, 0.00014756005839444697, 0.00014622247545048594, 0.00014465949789155275, 0.00014319346519187093, 0.00014180598373059183, 0.00014019827358424664, 0.00013833785487804562, 0.00013683992438018322, 0.0001354430423816666, 0.00013391274842433631, 0.0001325859921053052, 0.00013159512309357524, 0.00013019080506637692, 0.00012889757636003196, 0.00012779681128449738, 0.00012676592450588942, 0.00012570616672746837, 0.00012446215259842575, 0.00012316714855842292, 0.00012214825255796313, 0.00012107427755836397, 0.00011997888213954866, 0.00011879061639774591, 0.00011735207226593047, 0.0001163826382253319, 0.0001153902558144182, 0.00011439861555118114, 0.00011371332220733166, 0.00011326605454087257, 0.0001127346622524783 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00029742828155576717, "first_to_last_ratio": 2.2781890121807993, "imbalance_abs_log_ratio": 0.8233808344454435, "last_quartile_mean": 0.00013055469935352448, "mean": 0.00019329176166138495, "normalized": [ 1.8891850069388627, 1.7801603926271052, 1.6773544463608039, 1.566500262900107, 1.476456430986117, 1.3754925511314788, 1.3084504582544147, 1.236424592515421, 1.172965600373877, 1.150871569148191, 1.0802294364746856, 1.0319209662753672, 1.0080063247818516, 0.9698985570693867, 0.9067931019504507, 0.8756163462587222, 0.8444435806568995, 0.8113939671049883, 0.7809677247391495, 0.7613442367276676, 0.7402582678385915, 0.7293562882295931, 0.7162426695419845, 0.7062419232587185, 0.6940340549797547, 0.6866100033591334, 0.6831688143117768, 0.6753320518823956, 0.6715919825156708, 0.6686410475337363, 0.6640239865195754, 0.6600233567535221 ], "peak_layer": 1, "peak_normalized": 1.8891850069388627, "population_cv": 0.357915029863326, "rest_mean": 0.00020341356920888993, "spike_contrast": 0.6815376252628051, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001386340009048581, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00036516389809548855, 0.0003440903383307159, 0.0003242187958676368, 0.0003027915954589844, 0.00028538686456158757, 0.00026587137836031616, 0.0002529126941226423, 0.00023899068764876574, 0.00022672458726447076, 0.00022245399304665625, 0.0002087994507746771, 0.00019946182146668434, 0.00019483931828290224, 0.00018747340072877705, 0.00017527563613839447, 0.00016924942610785365, 0.00016322398732881993, 0.00015683576930314302, 0.0001509546273155138, 0.00014716156874783337, 0.0001430858246749267, 0.0001409785618307069, 0.00013844380737282336, 0.00013651074550580233, 0.00013415106514003128, 0.00013271605712361634, 0.00013205090363044292, 0.00013053612201474607, 0.00012981319741811603, 0.00012924280599690974, 0.00012835036613978446, 0.00012757707736454904 ] } } }, "pre_attention_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "de71d4cf2d509a1063f6eac7f50c4885e9a058dee7e4b993ad40f5527f528d00", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.02802751213312149, 0.01413465291261673, 0.014225807972252369, 0.01433455292135477, 0.01440478302538395, 0.009703479707241058, 0.00977881159633398, 0.009858808480203152, 0.009950309991836548, 0.00752287870272994, 0.007571155205368996, 0.007693085819482803, 0.007788818795233965, 0.006353048607707024, 0.006389216054230928, 0.0064707002602517605, 0.00656962301582098, 0.005553483963012695, 0.0056249531917274, 0.005688921548426151, 0.005768724251538515, 0.004966245498508215, 0.005057469941675663, 0.005098348017781973, 0.005216081161051989, 0.0046758754178881645, 0.004727107938379049, 0.004777040798217058, 0.004837613552808762, 0.0044186655431985855, 0.004510571248829365, 0.0045849340967834 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.0001580166108396952, "first_to_last_ratio": 1.9713740353643698, "imbalance_abs_log_ratio": 0.6787307795042399, "last_quartile_mean": 8.015557068574708e-05, "mean": 0.00011843875881822896, "normalized": [ 0.8918679597819287, 1.5814819593588525, 1.5722239987443252, 1.3246067564285042, 1.1252650527356378, 1.512743589128775, 1.3788112428745087, 1.2863035124436846, 1.097095055843533, 1.468154353173571, 1.1523254606068964, 1.0839888571909224, 1.054518217028451, 1.146335080549274, 1.1665208527784754, 0.8931136212023528, 0.8203804970547371, 1.073741223277434, 0.919654742394771, 0.8223971330190447, 0.7711288002244273, 0.8470775332584999, 0.8298711629738247, 0.7662487463744663, 0.7210382462310316, 0.7071376059209309, 0.7189559392079661, 0.727980518763353, 0.6111916361855597, 0.6986829946379066, 0.6244678178312321, 0.6046898327751223 ], "peak_layer": 2, "peak_normalized": 1.5814819593588525, "population_cv": 0.2904715201464359, "rest_mean": 0.00012310891098523926, "spike_contrast": 0.7572151875143335, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.321993711637333e-05, "top_five_layers": [ 2, 3, 6, 10, 7 ], "top_five_spike_overlap": 0 }, "values": [ 0.00010563173418631777, 0.0001873087603598833, 0.00018621225899551064, 0.00015688478015363216, 0.0001332749961875379, 0.00017916747310664505, 0.00016330469225067645, 0.00015234819147735834, 0.00012993857671972364, 0.00017388637934345752, 0.0001364799973089248, 0.0001283862948184833, 0.00012489582877606153, 0.00013577050413005054, 0.00013816128193866462, 0.00010577926877886057, 9.71648478298448e-05, 0.00012717257777694613, 0.00010892276623053476, 9.74036956904456e-05, 9.133153798757121e-05, 0.0001003268116619438, 9.828891052166e-05, 9.07535504666157e-05, 8.539887494407594e-05, 8.375250035896897e-05, 8.515224908478558e-05, 8.622110908618197e-05, 7.238877878990024e-05, 8.275114669231698e-05, 7.396119326585904e-05, 7.161871326388791e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.0001664285200604354, "first_to_last_ratio": 1.9719337364651075, "imbalance_abs_log_ratio": 0.6790146534176558, "last_quartile_mean": 8.439863722742302e-05, "mean": 0.0001245867656507471, "normalized": [ 0.8984334461398115, 1.5927952483854981, 1.5658595485525297, 1.3198735865176374, 1.131213894321221, 1.5125315092450449, 1.3796810696180641, 1.286366052942558, 1.0960606282761096, 1.4536884557404346, 1.1543342746982923, 1.0797559630746076, 1.0497530570909623, 1.1457182987306291, 1.1551962694199982, 0.9023721663305979, 0.8238710779144349, 1.0632503993742801, 0.9211624832492573, 0.8274335806257342, 0.7739032412019898, 0.850990856837025, 0.8281317613098801, 0.7681943949939335, 0.7208251570035091, 0.7089546837555952, 0.7203162530736348, 0.7236174125342815, 0.6145740290254166, 0.6994991973458914, 0.6274381906330746, 0.6042038120380651 ], "peak_layer": 2, "peak_normalized": 1.5927952483854981, "population_cv": 0.2889665774838781, "rest_mean": 0.00012946851159169995, "spike_contrast": 0.7586812913967177, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.822533756960183e-05, "top_five_layers": [ 2, 3, 6, 10, 7 ], "top_five_spike_overlap": 0 }, "values": [ 0.00011193291720701382, 0.0001984412083402276, 0.0001950853766174987, 0.000164438781212084, 0.00014093428035266697, 0.00018844140868168324, 0.00017189000209327787, 0.00016026418597903103, 0.00013655464863404632, 0.00018111034296453, 0.00014381477376446128, 0.0001345233031315729, 0.00013078533811494708, 0.00014274133718572557, 0.00014392216689884663, 0.00011242362961638719, 0.00010264343291055411, 0.0001324669283349067, 0.00011476465442683548, 0.00010308727360097691, 9.641810174798593e-05, 0.00010602219845168293, 0.00010317425767425448, 9.570685506332666e-05, 8.980527491075918e-05, 8.832637104205787e-05, 8.974187221610919e-05, 9.015315299620852e-05, 7.656779052922502e-05, 8.714834257261828e-05, 7.817049481673166e-05, 7.527579873567447e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.14759047608822584, "first_to_last_ratio": 1.9719337947467823, "imbalance_abs_log_ratio": 0.6790146829732503, "last_quartile_mean": 0.07484555337578058, "mean": 0.1104847895912826, "normalized": [ 0.8984334974418968, 1.5927952138130326, 1.565859646322909, 1.3198735575362357, 1.1312139236939245, 1.5125315007911495, 1.3796810264894532, 1.286366010451999, 1.0960606168617568, 1.4536884922319049, 1.1543342013148132, 1.0797558953237962, 1.0497530983161607, 1.1457182562234747, 1.1551962948455465, 0.9023721940039697, 0.8238711061937815, 1.0632504861804335, 0.9211625151546727, 0.8274335809310764, 0.7739032642161865, 0.8509909070635955, 0.8281317391228538, 0.7681943896830757, 0.7208251689669645, 0.7089546589522648, 0.7203162343708721, 0.7236173971219663, 0.6145740294939214, 0.6994991437375198, 0.6274381319743458, 0.6042038211744466 ], "peak_layer": 2, "peak_normalized": 1.5927952138130326, "population_cv": 0.28896658018674404, "rest_mean": 0.11481397047086998, "spike_contrast": 0.7586813040631772, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.08710721284151077, "top_five_layers": [ 2, 3, 6, 10, 7 ], "top_five_spike_overlap": 0 }, "values": [ 0.09926323592662811, 0.1759796440601349, 0.1730036735534668, 0.14582595229148865, 0.12498193234205246, 0.16711172461509705, 0.15243376791477203, 0.14212387800216675, 0.12109802663326263, 0.16061046719551086, 0.1275363713502884, 0.1192966029047966, 0.11598175019025803, 0.12658444046974182, 0.12763161957263947, 0.09969840198755264, 0.0910252258181572, 0.11747300624847412, 0.10177444666624069, 0.09141882508993149, 0.08550453931093216, 0.09402155131101608, 0.09149596095085144, 0.08487379550933838, 0.0796402171254158, 0.0783287063241005, 0.07958398759365082, 0.07994871586561203, 0.06790108233690262, 0.07728401571512222, 0.069322369992733, 0.06675533205270767 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 5.093040158499207e-05, "first_to_last_ratio": 0.8860093039525244, "imbalance_abs_log_ratio": 0.12102782735710801, "last_quartile_mean": 5.748291960117058e-05, "mean": 5.7093861016710434e-05, "normalized": [ 0.4337136137178416, 0.9185569667519601, 0.9444095926066538, 0.8728787927927439, 0.7849140301431445, 1.1296292591692207, 1.0726593944235638, 0.9796135302182263, 0.9386642177229508, 1.3046353648934683, 1.0461832118506482, 1.0032917361864164, 0.9662905330123907, 1.0760080966276753, 1.1828739682986207, 0.9087955578115051, 0.9194941496879511, 1.198292373503509, 1.0946132296674007, 1.0673695485587213, 0.9244260207272295, 1.0525679458300632, 1.125653431968099, 0.9999504891362015, 0.9629027708994021, 1.0063234987710272, 0.9975933861195996, 1.0791525148711678, 0.9412468232151083, 1.085173475571988, 1.0031556955576197, 0.9789667796878816 ], "peak_layer": 10, "peak_normalized": 1.3046353648934683, "population_cv": 0.1441687540611157, "rest_mean": 5.695535414647397e-05, "spike_contrast": 1.0155638391297448, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.784179811598733e-05, "top_five_layers": [ 10, 18, 15, 6, 23 ], "top_five_spike_overlap": 1 }, "values": [ 2.4762384782661684e-05, 5.2443963795667514e-05, 5.3919990023132414e-05, 4.9836020480142906e-05, 4.481377254705876e-05, 6.449489592341706e-05, 6.124226638348773e-05, 5.592991874436848e-05, 5.3591964388033375e-05, 7.448667020071298e-05, 5.973063889541663e-05, 5.728179894504137e-05, 5.516925739357248e-05, 6.143345672171563e-05, 6.75348419463262e-05, 5.1886647270293906e-05, 5.249747118796222e-05, 6.84151382301934e-05, 6.249569560168311e-05, 6.094024865888059e-05, 5.277905074763112e-05, 6.009516800986603e-05, 6.426790059776977e-05, 5.7091034250333905e-05, 5.4975836974335834e-05, 5.74548939766828e-05, 5.695645813830197e-05, 6.161298369988799e-05, 5.373941530706361e-05, 6.19567435933277e-05, 5.727403186028823e-05, 5.58929932594765e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 5.801741053801379e-05, "first_to_last_ratio": 0.954195857595535, "imbalance_abs_log_ratio": 0.046886327142486896, "last_quartile_mean": 6.08024129178375e-05, "mean": 6.296396202287724e-05, "normalized": [ 0.4796364558374551, 0.9568353419130547, 0.9804638447804884, 0.8949545354119591, 0.8237377234766136, 1.1597846916175047, 1.085649359633547, 0.9904450076993521, 0.9595078978373608, 1.2941748206155441, 1.0680753760655664, 1.0231559828609242, 0.9904019047543832, 1.096490963646882, 1.1850351594596897, 0.9422215946487361, 0.9418631353047864, 1.1705024179981143, 1.0926353878857977, 1.0382275689133567, 0.9320833313178482, 1.0784583323883348, 1.0985080196909318, 0.9917906747078287, 0.9476388143731139, 0.9855066582268529, 0.978396290111904, 1.028407839812513, 0.8982055143685478, 1.0282804954549911, 0.9451124964564701, 0.9138123627295472 ], "peak_layer": 10, "peak_normalized": 1.2941748206155441, "population_cv": 0.13431233496292436, "rest_mean": 6.285090866084729e-05, "spike_contrast": 1.0115120295379672, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.357445017783903e-05, "top_five_layers": [ 10, 15, 18, 6, 23 ], "top_five_spike_overlap": 1 }, "values": [ 3.019981159013696e-05, 6.0246144130360335e-05, 6.173388828756288e-05, 5.634988337988034e-05, 5.186579073779285e-05, 7.302463927771896e-05, 6.835678505012766e-05, 6.236234185053036e-05, 6.041441884008236e-05, 8.148637425620109e-05, 6.725025741616264e-05, 6.442195444833487e-05, 6.235962791834027e-05, 6.903941539349034e-05, 7.461450877599418e-05, 5.9326004702597857e-05, 5.930343468207866e-05, 7.369946979451925e-05, 6.879665306769311e-05, 6.537092122016475e-05, 5.8687659475253895e-05, 6.790400948375463e-05, 6.916641723364592e-05, 6.244707037694752e-05, 5.966709431959316e-05, 6.205140380188823e-05, 6.160370685392991e-05, 6.475263216998428e-05, 5.655457789544016e-05, 6.474461406469345e-05, 5.950802733423188e-05, 5.753724690293893e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 1.7888660295284353e-05, "first_to_last_ratio": 0.41855462748958633, "imbalance_abs_log_ratio": 0.8709478659353102, "last_quartile_mean": 4.2739129185065394e-05, "mean": 3.1986895351110434e-05, "normalized": [ 0.2593955501951102, 0.4997829486650695, 0.48480994473028793, 0.4968760363401999, 0.5474125689219601, 0.7788406177150305, 0.7554473604850604, 0.6514322393717875, 0.7578135841604356, 0.9674810188795085, 0.9032772556875033, 0.9019220651946442, 0.8311356963419073, 0.9587970177975769, 1.1401394869948454, 0.8953886492354074, 1.0262272842074007, 1.1413916918814533, 1.174037958462901, 1.194927715786966, 1.0267140632187253, 1.3434898267078448, 1.3729067467361273, 1.2011929483563188, 1.1735959904166378, 1.3266745689353452, 1.2739952988364465, 1.4049535244931144, 1.2752889026856997, 1.4680979746684197, 1.4182292846928506, 1.3483241791974143 ], "peak_layer": 30, "peak_normalized": 1.4680979746684197, "population_cv": 0.3184955307507873, "rest_mean": 3.066251991657299e-05, "spike_contrast": 1.276428773763581, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.913852269761264e-05, "top_five_layers": [ 30, 31, 28, 23, 32 ], "top_five_spike_overlap": 1 }, "values": [ 8.297258318634704e-06, 1.5986504877218977e-05, 1.5507564967265353e-05, 1.589352177688852e-05, 1.7510028555989265e-05, 2.491269333404489e-05, 2.416441566310823e-05, 2.083729486912489e-05, 2.4240103812189773e-05, 3.094671410508454e-05, 2.889303505071439e-05, 2.8849686714238487e-05, 2.6585450541460887e-05, 3.066893987124786e-05, 3.6469522456172854e-05, 2.8640703021665104e-05, 3.282582474639639e-05, 3.6509576602838933e-05, 3.755382931558415e-05, 3.822202779701911e-05, 3.2841395295690745e-05, 4.2974068492185324e-05, 4.391502443468198e-05, 3.842243313556537e-05, 3.7539692129939795e-05, 4.243620060151443e-05, 4.0751154301688075e-05, 4.494010136113502e-05, 4.0792532672639936e-05, 4.695989628089592e-05, 4.5364751713350415e-05, 4.3128704419359565e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00022091567916504573, "first_to_last_ratio": 1.232986382535498, "imbalance_abs_log_ratio": 0.20943917994887448, "last_quartile_mean": 0.00017917122386279516, "mean": 0.0002198539991695725, "normalized": [ 0.4672441326905094, 1.0299890566632133, 1.08986441571188, 1.0084939820045402, 0.8814237539069011, 1.2693826047962031, 1.1886514905905403, 1.1035827467935222, 1.0473307057033197, 1.4274044616309502, 1.1442998336482333, 1.0911283607583675, 1.1094038041384326, 1.1946873974128753, 1.2511466100570947, 0.9923165311956169, 0.9239703707694655, 1.2473578196113966, 1.1181336030529359, 1.0160745414007064, 0.9397394354798955, 0.9905446518023159, 1.0121913654482901, 0.9359943959598221, 0.8839446941554127, 0.8510229499852975, 0.8648513533654478, 0.8786898836618169, 0.7455559493639876, 0.8399702482950041, 0.7392957942269532, 0.7163130557190521 ], "peak_layer": 10, "peak_normalized": 1.4274044616309502, "population_cv": 0.18952709251819305, "rest_mean": 0.0002217885959436427, "spike_contrast": 0.9441746799407325, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00020940717658959329, "top_five_layers": [ 10, 6, 15, 18, 14 ], "top_five_spike_overlap": 0 }, "values": [ 0.00010272549116052687, 0.00022644721320830286, 0.00023961105034686625, 0.00022172143508214504, 0.0001937845372594893, 0.0002790788421407342, 0.0002613297838252038, 0.00024262708029709756, 0.00023025984410196543, 0.000313820579322055, 0.00025157889467664063, 0.00023988893372006714, 0.00024390686303377151, 0.000262656802078709, 0.00027506958576850593, 0.0002181647578254342, 0.00020313858112785965, 0.00027423660503700376, 0.0002458261442370713, 0.00022338805138133466, 0.00020660547306761146, 0.00021777520305477083, 0.00022253431961871684, 0.00020578211115207523, 0.0001943387760547921, 0.00018710079893935472, 0.0001901410287246108, 0.00019318348495289683, 0.00016391345707233995, 0.00018467081827111542, 0.00016253713693004102, 0.00015748428995721042 ] } } }, "pre_mlp_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "2999a257265024350fd2665e2a8d452a03cd662193b4245d15290408ad29922d", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.014061377383768559, 0.014165290631353855, 0.014281988143920898, 0.014350161887705326, 0.00966146495193243, 0.009741888381540775, 0.009820535778999329, 0.00991273857653141, 0.0074927168898284435, 0.007546746172010899, 0.00766373984515667, 0.007762390188872814, 0.006325984839349985, 0.006375366821885109, 0.006445344537496567, 0.006555322092026472, 0.005531130358576775, 0.005611324217170477, 0.0056777093559503555, 0.00574630219489336, 0.004946626722812653, 0.0050392793491482735, 0.005084683652967215, 0.005203457549214363, 0.004667261149734259, 0.004717131145298481, 0.004768412560224533, 0.004826710093766451, 0.0044052014127373695, 0.004504534415900707, 0.004574235528707504, 0.004644096828997135 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 3.729895206561196e-05, "first_to_last_ratio": 1.2469721805352876, "imbalance_abs_log_ratio": 0.22071835733540474, "last_quartile_mean": 2.9911615229138988e-05, "mean": 3.340667609563752e-05, "normalized": [ 1.308752778077389, 1.2456464596610868, 1.0236956931411572, 0.9922285647386262, 1.26903181062375, 1.1170846373362797, 1.0287316533628064, 0.9469237624012169, 1.1300261751959062, 1.1132698789554223, 1.0126240729566007, 0.9379202574439188, 1.1130503370514597, 1.070713488608195, 0.9897642722064988, 0.922085688720865, 1.036831181462568, 0.9898105546019274, 0.9395130252451869, 0.9001967292772246, 0.9975952535130206, 0.9404117749145336, 0.9224862220158919, 0.8885787590271026, 0.9461670180110204, 0.8974372095121397, 0.8904691498559152, 0.8684437934704528, 0.9244147277585869, 0.9015689206268677, 0.8740901912628669, 0.8604359589635168 ], "peak_layer": 1, "peak_normalized": 1.308752778077389, "population_cv": 0.1164905978879517, "rest_mean": 3.378375206213807e-05, "spike_contrast": 0.9285666618330386, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.137046587653458e-05, "top_five_layers": [ 1, 5, 2, 9, 6 ], "top_five_spike_overlap": 0 }, "values": [ 4.3721080146497115e-05, 4.161290780757554e-05, 3.419827044126578e-05, 3.314705827506259e-05, 4.239413465256803e-05, 3.73180846509058e-05, 3.4366505133220926e-05, 3.1633575417799875e-05, 3.7750418414361775e-05, 3.719064625329338e-05, 3.3828404411906376e-05, 3.1332798243965954e-05, 3.718331208801828e-05, 3.5768978705164045e-05, 3.306473445263691e-05, 3.080381793552078e-05, 3.463708344497718e-05, 3.3066280593629926e-05, 3.138600732199848e-05, 3.007258055731654e-05, 3.332634150865488e-05, 3.1416031561093405e-05, 3.0817198421573266e-05, 2.9684462788281962e-05, 3.1608295103069395e-05, 2.9980394174344838e-05, 2.9747614462394267e-05, 2.9011820515734144e-05, 3.0881623388268054e-05, 3.01184209092753e-05, 2.9200447897892445e-05, 2.874430538213346e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 6.705871965095866e-05, "first_to_last_ratio": 0.7180297289018083, "imbalance_abs_log_ratio": 0.33124430563536084, "last_quartile_mean": 9.33926785364747e-05, "mean": 8.228966771639534e-05, "normalized": [ 0.8685826408392191, 0.8249006353131848, 0.7154905339803773, 0.6884730939760333, 0.9470487095509537, 0.8638069620260335, 0.8256155016985178, 0.7853664463613442, 0.9851686390305121, 0.9692511243547932, 0.9083804947955202, 0.8729855457896263, 1.0422548534804916, 1.0255492616133721, 0.9871528463964067, 0.9398223255303129, 1.0958375595009802, 1.0726325633807163, 1.0495259774381736, 1.0230838788771643, 1.1576441025653552, 1.1142532603054807, 1.1008567438213033, 1.056908774400763, 1.159063578749664, 1.1192434432436902, 1.1032241584740294, 1.1060435702772593, 1.1786235078649367, 1.1628365878275773, 1.141840825041133, 1.1085318534950752 ], "peak_layer": 29, "peak_normalized": 1.1786235078649367, "population_cv": 0.1341844256762087, "rest_mean": 8.049536754023629e-05, "spike_contrast": 1.1426606459269575, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.197888866765424e-05, "top_five_layers": [ 29, 30, 25, 21, 31 ], "top_five_spike_overlap": 2 }, "values": [ 7.14753768988885e-05, 6.788079917896539e-05, 5.887747829547152e-05, 5.66542221349664e-05, 7.793232362018898e-05, 7.108238787623122e-05, 6.793962529627606e-05, 6.462754390668124e-05, 8.106919995043427e-05, 7.975935295689851e-05, 7.475032907677814e-05, 7.183769048424438e-05, 8.576680556870997e-05, 8.4392107964959e-05, 8.123247971525416e-05, 7.733766688033938e-05, 9.017610864248127e-05, 8.826657722238451e-05, 8.636514394311234e-05, 8.418923243880272e-05, 9.526214853394777e-05, 9.169153054244816e-05, 9.0589135652408e-05, 8.697267185198143e-05, 9.537895675748587e-05, 9.210217103827745e-05, 9.078394941752777e-05, 9.101595787797123e-05, 9.698853682493791e-05, 9.56894364207983e-05, 9.396170207764953e-05, 9.122071787714958e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.05946834245696664, "first_to_last_ratio": 0.7180297222365345, "imbalance_abs_log_ratio": 0.33124431491808765, "last_quartile_mean": 0.08282156102359295, "mean": 0.07297530013602227, "normalized": [ 0.8685826122359729, 0.8249005917343738, 0.71549052911611, 0.6884731350635278, 0.9470486699630107, 0.8638070113577097, 0.8256154769796886, 0.7853664269052657, 0.9851686341386658, 0.9692511556795396, 0.9083805454214465, 0.8729855579949275, 1.0422548040385637, 1.0255492365750694, 0.9871528950273681, 0.9398223256645052, 1.0958375053304956, 1.07263252629438, 1.0495259690486791, 1.0230837911446407, 1.1576441435704576, 1.1142533045483842, 1.1008568124470575, 1.0569088284972943, 1.1590636022344982, 1.119243411839159, 1.1032241423789504, 1.1060435591240758, 1.1786235037173105, 1.1628366076958818, 1.141840809884862, 1.1085318743481278 ], "peak_layer": 29, "peak_normalized": 1.1786235037173105, "population_cv": 0.13418442668798974, "rest_mean": 0.07138409617322462, "spike_contrast": 1.1426607032635474, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.08156780153512955, "top_five_layers": [ 29, 30, 25, 21, 31 ], "top_five_spike_overlap": 2 }, "values": [ 0.06338507682085037, 0.0601973682641983, 0.05221313610672951, 0.050241533666849136, 0.0691111609339714, 0.06303657591342926, 0.060249537229537964, 0.05731235072016716, 0.07189297676086426, 0.07073139399290085, 0.06628934293985367, 0.06370638310909271, 0.07605885714292526, 0.0748397633433342, 0.07203777879476547, 0.06858381628990173, 0.07996907085180283, 0.07827568054199219, 0.07658947259187698, 0.07465984672307968, 0.0844794288277626, 0.08131296932697296, 0.0803353562951088, 0.07712823897600174, 0.08458301424980164, 0.08167712390422821, 0.08050811290740967, 0.08071386069059372, 0.0860104039311409, 0.08485835045576096, 0.08332617580890656, 0.08089544624090195 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 7.408341275549901e-06, "first_to_last_ratio": 0.49124437919492936, "imbalance_abs_log_ratio": 0.7108135577065074, "last_quartile_mean": 1.5080765479069669e-05, "mean": 1.1844782648040564e-05, "normalized": [ 0.5657990507718705, 0.5382233871697658, 0.49942145823009426, 0.48906984415931193, 0.8062921756393574, 0.7122953827323945, 0.7013508394120195, 0.6911626993599347, 0.8940675881372303, 0.868722127857376, 0.9735911331661544, 0.854855860776803, 0.989483587379368, 1.0059088486455374, 1.0303729006301217, 0.9941025538400318, 1.1620912657632405, 1.127012997009919, 1.113401270263767, 1.066097770938846, 1.1826034542812638, 1.1643547935214302, 1.2049778184223279, 1.1791491538213539, 1.2365172470752577, 1.219921447624247, 1.2337761202582145, 1.2288278254433027, 1.3039217543779833, 1.3631114810840155, 1.2856635723507386, 1.3138525898567213 ], "peak_layer": 30, "peak_normalized": 1.3631114810840155, "population_cv": 0.2549101540724733, "rest_mean": 1.1420299651271543e-05, "spike_contrast": 1.237882653019464, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.4136990830593277e-05, "top_five_layers": [ 30, 32, 29, 31, 25 ], "top_five_spike_overlap": 1 }, "values": [ 6.701766778860474e-06, 6.3751390371180605e-06, 5.915538622502936e-06, 5.7929260037781205e-06, 9.550355571263935e-06, 8.436983989668079e-06, 8.307348252856173e-06, 8.186671948351432e-06, 1.0590036254143342e-05, 1.0289824786013924e-05, 1.1531975360412616e-05, 1.0125581866304856e-05, 1.1720218026312068e-05, 1.1914771675947122e-05, 1.220454305439489e-05, 1.1774928680097219e-05, 1.3764718460151926e-05, 1.334922399109928e-05, 1.3187996046326589e-05, 1.2627696378331166e-05, 1.4007680874783546e-05, 1.379152945446549e-05, 1.4272700354922563e-05, 1.3966765436634887e-05, 1.4646278032159898e-05, 1.4449704394792207e-05, 1.4613809980801307e-05, 1.4555198504240252e-05, 1.5444669770658948e-05, 1.614575921848882e-05, 1.5228405572997872e-05, 1.5562298358418047e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 5.6029601637419546e-05, "first_to_last_ratio": 0.6070213107493427, "imbalance_abs_log_ratio": 0.49919138022039955, "last_quartile_mean": 9.230252817360451e-05, "mean": 7.840498903988191e-05, "normalized": [ 0.6598133090619259, 0.6419677920829977, 0.6005256373772865, 0.5913463573822231, 0.8562376513989928, 0.8062968559250673, 0.7923156566726156, 0.768439227921366, 0.9790122729268586, 0.9665730330981341, 0.9163710065350301, 0.8881686280668529, 1.0631190324760253, 1.0511030519715439, 1.0188080221730096, 0.9712812246823179, 1.1361272291219966, 1.1138561419734825, 1.0918134348286874, 1.0639217496772315, 1.203414696914147, 1.1592426077267617, 1.1442664103435203, 1.097952968622282, 1.2042281788778202, 1.1625445130141736, 1.1452986211445626, 1.1473932954411665, 1.2233942816523278, 1.205318760675343, 1.1829469858778916, 1.146901364356358 ], "peak_layer": 29, "peak_normalized": 1.2233942816523278, "population_cv": 0.1898597249088972, "rest_mean": 7.605543874820728e-05, "spike_contrast": 1.1977126437531995, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.109256061492487e-05, "top_five_layers": [ 29, 30, 25, 21, 31 ], "top_five_spike_overlap": 2 }, "values": [ 5.173265526536852e-05, 5.033347770222463e-05, 4.708420601673424e-05, 4.636450466932729e-05, 6.713330367347226e-05, 6.321769615169615e-05, 6.212150037754327e-05, 6.024946924299002e-05, 7.675944652874023e-05, 7.578414806630462e-05, 7.184805872384459e-05, 6.963685154914856e-05, 8.335383608937263e-05, 8.241172326961532e-05, 7.987963181221858e-05, 7.615329377586022e-05, 8.907804294722155e-05, 8.733187860343605e-05, 8.560362039133906e-05, 8.341677312273532e-05, 9.435371612198651e-05, 9.089040395338088e-05, 8.971619536168873e-05, 8.608499047113582e-05, 9.441749716643244e-05, 9.114928980125114e-05, 8.97971258382313e-05, 8.996135875349864e-05, 9.592021524440497e-05, 9.450300422031432e-05, 9.274894546251744e-05, 8.992278890218586e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 4.8511469685763586e-05, "first_to_last_ratio": 0.5271352332510719, "imbalance_abs_log_ratio": 0.6402981537748061, "last_quartile_mean": 9.202850924339145e-05, "mean": 7.532146378252946e-05, "normalized": [ 0.548104082267993, 0.5416830199005969, 0.528884702689083, 0.5247448675895723, 0.7777813723908718, 0.7487613756360456, 0.7496059866168319, 0.7329069161583202, 0.9480676391902623, 0.9399123881532756, 0.9000936195631334, 0.883441930780847, 1.0560589316963704, 1.0534552095740957, 1.032095781768693, 0.9835620691098637, 1.1601754560133901, 1.1467788545782247, 1.1233340534265408, 1.0967375255670042, 1.2396083970871608, 1.1951467072205837, 1.1828066060767042, 1.1317734013234224, 1.244698956870482, 1.2025862582796845, 1.1881970083126314, 1.1882256015362047, 1.2689460104605979, 1.2589162611011824, 1.2307315535498626, 1.1921774555104672 ], "peak_layer": 29, "peak_normalized": 1.2689460104605979, "population_cv": 0.23486334953575583, "rest_mean": 7.254842300123225e-05, "spike_contrast": 1.244629176846487, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.029588400153444e-05, "top_five_layers": [ 29, 30, 25, 21, 31 ], "top_five_spike_overlap": 2 }, "values": [ 4.1284001781605184e-05, 4.080035796505399e-05, 3.9836369978729635e-05, 3.952455153921619e-05, 5.858363147126511e-05, 5.639780283672735e-05, 5.6461420172126964e-05, 5.520362174138427e-05, 7.140984234865755e-05, 7.079557690303773e-05, 6.77963689668104e-05, 6.654213939327747e-05, 7.954390457598493e-05, 7.934778841445222e-05, 7.773896504659206e-05, 7.408333476632833e-05, 8.738611359149218e-05, 8.637706196168438e-05, 8.461116522084922e-05, 8.26078758109361e-05, 9.336911898571998e-05, 9.002019942272455e-05, 8.909072494134307e-05, 8.524682925781235e-05, 9.375254740007222e-05, 9.058055729838088e-05, 8.949673792812973e-05, 8.949889161158353e-05, 9.557887096889317e-05, 9.482341556577012e-05, 9.270050213672221e-05, 8.979655103757977e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 9.61669284151867e-05, "first_to_last_ratio": 0.8297833624837898, "imbalance_abs_log_ratio": 0.18659062131596024, "last_quartile_mean": 0.00011589401856326731, "mean": 0.00010716335873439675, "normalized": [ 0.9380742209455786, 0.8898820858744699, 0.7837417114644826, 0.7583102664636657, 1.0822070619439073, 0.9520749081713696, 0.9107982467806989, 0.8640017807748708, 1.0669538303069324, 1.0266950649381863, 0.9594029855154438, 0.8901506143407661, 1.0795289063552718, 1.0379165674077444, 0.9706971366471856, 0.914979822300863, 1.052939971267624, 1.0225707935964177, 0.9851238030379684, 0.9636652893153618, 1.0990044510799908, 1.0523904893769196, 1.0422234788441622, 1.0049019599938585, 1.0962247910521359, 1.0599378706264646, 1.05039930496957, 1.0583562345651827, 1.1199788667078585, 1.1100529546792268, 1.0850973923636869, 1.0717171382921347 ], "peak_layer": 29, "peak_normalized": 1.1199788667078585, "population_cv": 0.09156717959150794, "rest_mean": 0.00010599351123716958, "spike_contrast": 1.0706366257223145, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00011348053521942347, "top_five_layers": [ 29, 30, 21, 25, 31 ], "top_five_spike_overlap": 2 }, "values": [ 0.00010052718425868079, 9.536275319987908e-05, 8.398839418077841e-05, 8.12630751170218e-05, 0.00011597294360399246, 0.00010202754492638633, 9.760419925441965e-05, 9.25893327803351e-05, 0.00011433835607022047, 0.00011002409155480564, 0.00010281284630764276, 9.539152961224318e-05, 0.000115685943455901, 0.0001112266254494898, 0.0001040231654769741, 9.805231093196198e-05, 0.00011283658386673778, 0.00010958212078548968, 0.00010556917550275102, 0.00010326960909878835, 0.00011777300824178383, 0.0001127776995417662, 0.00011168816854478791, 0.00010768866923172027, 0.00011747513053705916, 0.00011358650226611644, 0.00011256431753281504, 0.00011341700883349404, 0.00012002069706795737, 0.00011895700299646705, 0.00011628268111962825, 0.00011484880815260112 ] } } } }, "selector": { "expected_uniform_count": 4, "identities_unique": true, "identity_order_sha256": "97fd9721a263024f6a746638f32e0ad9a988cd4c111c00e9e178c1022e5c9d5e", "passed": true, "uniform_count": 4, "uniform_indices": [ 48, 50, 52, 54 ], "visit_count": 65, "visits": [ { "branch": "attention", "coefficient": "detached_learned", "group": 1, "index": 0, "kind": "depth", "layer": 1, "offset": 1, "sources": 1, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 1, "index": 1, "kind": "depth", "layer": 1, "offset": 1, "sources": 2, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 1, "index": 2, "kind": "depth", "layer": 2, "offset": 2, "sources": 2, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 1, "index": 3, "kind": "depth", "layer": 2, "offset": 2, "sources": 2, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 1, "index": 4, "kind": "depth", "layer": 3, "offset": 3, "sources": 2, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 1, "index": 5, "kind": "depth", "layer": 3, "offset": 3, "sources": 2, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 1, "index": 6, "kind": "depth", "layer": 4, "offset": 4, "sources": 2, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 1, "index": 7, "kind": "depth", "layer": 4, "offset": 4, "sources": 2, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 2, "index": 8, "kind": "depth", "layer": 5, "offset": 1, "sources": 2, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 2, "index": 9, "kind": "depth", "layer": 5, "offset": 1, "sources": 3, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 2, "index": 10, "kind": "depth", "layer": 6, "offset": 2, "sources": 3, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 2, "index": 11, "kind": "depth", "layer": 6, "offset": 2, "sources": 3, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 2, "index": 12, "kind": "depth", "layer": 7, "offset": 3, "sources": 3, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 2, "index": 13, "kind": "depth", "layer": 7, "offset": 3, "sources": 3, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 2, "index": 14, "kind": "depth", "layer": 8, "offset": 4, "sources": 3, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 2, "index": 15, "kind": "depth", "layer": 8, "offset": 4, "sources": 3, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 3, "index": 16, "kind": "depth", "layer": 9, "offset": 1, "sources": 3, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 3, "index": 17, "kind": "depth", "layer": 9, "offset": 1, "sources": 4, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 3, "index": 18, "kind": "depth", "layer": 10, "offset": 2, "sources": 4, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 3, "index": 19, "kind": "depth", "layer": 10, "offset": 2, "sources": 4, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 3, "index": 20, "kind": "depth", "layer": 11, "offset": 3, "sources": 4, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 3, "index": 21, "kind": "depth", "layer": 11, "offset": 3, "sources": 4, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 3, "index": 22, "kind": "depth", "layer": 12, "offset": 4, "sources": 4, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 3, "index": 23, "kind": "depth", "layer": 12, "offset": 4, "sources": 4, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 4, "index": 24, "kind": "depth", "layer": 13, "offset": 1, "sources": 4, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 4, "index": 25, "kind": "depth", "layer": 13, "offset": 1, "sources": 5, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 4, "index": 26, "kind": "depth", "layer": 14, "offset": 2, "sources": 5, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 4, "index": 27, "kind": "depth", "layer": 14, "offset": 2, "sources": 5, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 4, "index": 28, "kind": "depth", "layer": 15, "offset": 3, "sources": 5, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 4, "index": 29, "kind": "depth", "layer": 15, "offset": 3, "sources": 5, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 4, "index": 30, "kind": "depth", "layer": 16, "offset": 4, "sources": 5, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 4, "index": 31, "kind": "depth", "layer": 16, "offset": 4, "sources": 5, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 5, "index": 32, "kind": "depth", "layer": 17, "offset": 1, "sources": 5, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 5, "index": 33, "kind": "depth", "layer": 17, "offset": 1, "sources": 6, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 5, "index": 34, "kind": "depth", "layer": 18, "offset": 2, "sources": 6, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 5, "index": 35, "kind": "depth", "layer": 18, "offset": 2, "sources": 6, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 5, "index": 36, "kind": "depth", "layer": 19, "offset": 3, "sources": 6, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 5, "index": 37, "kind": "depth", "layer": 19, "offset": 3, "sources": 6, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 5, "index": 38, "kind": "depth", "layer": 20, "offset": 4, "sources": 6, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 5, "index": 39, "kind": "depth", "layer": 20, "offset": 4, "sources": 6, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 6, "index": 40, "kind": "depth", "layer": 21, "offset": 1, "sources": 6, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 6, "index": 41, "kind": "depth", "layer": 21, "offset": 1, "sources": 7, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 6, "index": 42, "kind": "depth", "layer": 22, "offset": 2, "sources": 7, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 6, "index": 43, "kind": "depth", "layer": 22, "offset": 2, "sources": 7, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 6, "index": 44, "kind": "depth", "layer": 23, "offset": 3, "sources": 7, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 6, "index": 45, "kind": "depth", "layer": 23, "offset": 3, "sources": 7, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 6, "index": 46, "kind": "depth", "layer": 24, "offset": 4, "sources": 7, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 6, "index": 47, "kind": "depth", "layer": 24, "offset": 4, "sources": 7, "use_uniform": false }, { "branch": "attention", "coefficient": "uniform", "group": 7, "index": 48, "kind": "depth", "layer": 25, "offset": 1, "sources": 7, "use_uniform": true }, { "branch": "mlp", "coefficient": "detached_learned", "group": 7, "index": 49, "kind": "depth", "layer": 25, "offset": 1, "sources": 8, "use_uniform": false }, { "branch": "attention", "coefficient": "uniform", "group": 7, "index": 50, "kind": "depth", "layer": 26, "offset": 2, "sources": 8, "use_uniform": true }, { "branch": "mlp", "coefficient": "detached_learned", "group": 7, "index": 51, "kind": "depth", "layer": 26, "offset": 2, "sources": 8, "use_uniform": false }, { "branch": "attention", "coefficient": "uniform", "group": 7, "index": 52, "kind": "depth", "layer": 27, "offset": 3, "sources": 8, "use_uniform": true }, { "branch": "mlp", "coefficient": "detached_learned", "group": 7, "index": 53, "kind": "depth", "layer": 27, "offset": 3, "sources": 8, "use_uniform": false }, { "branch": "attention", "coefficient": "uniform", "group": 7, "index": 54, "kind": "depth", "layer": 28, "offset": 4, "sources": 8, "use_uniform": true }, { "branch": "mlp", "coefficient": "detached_learned", "group": 7, "index": 55, "kind": "depth", "layer": 28, "offset": 4, "sources": 8, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 8, "index": 56, "kind": "depth", "layer": 29, "offset": 1, "sources": 8, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 8, "index": 57, "kind": "depth", "layer": 29, "offset": 1, "sources": 9, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 8, "index": 58, "kind": "depth", "layer": 30, "offset": 2, "sources": 9, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 8, "index": 59, "kind": "depth", "layer": 30, "offset": 2, "sources": 9, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 8, "index": 60, "kind": "depth", "layer": 31, "offset": 3, "sources": 9, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 8, "index": 61, "kind": "depth", "layer": 31, "offset": 3, "sources": 9, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 8, "index": 62, "kind": "depth", "layer": 32, "offset": 4, "sources": 9, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 8, "index": 63, "kind": "depth", "layer": 32, "offset": 4, "sources": 9, "use_uniform": false }, { "branch": "output", "coefficient": "detached_learned", "group": null, "index": 64, "kind": "output", "layer": null, "offset": null, "sources": 9, "use_uniform": false } ], "visits_sha256": "67a61506afe78423707af99e34dfe341cf023966da8e40742d204c35f4eb2249" } }, "uniform_group_7_mlp_only": { "bits_per_byte": 7.959341947457508, "forward": { "activation_sha256": { "attention_branch_output": "676759a2d7c0615ad7e1f246e90927fb829d00c8b6c3745032ae6b27ab7d902a", "mlp_branch_output": "afe29f3ef4d2992353f0bb59fe7c5c2c5050fca671a53184825a8d63cd83a2bc", "post_attention_state": "31853ffb07b24cc59e774bb20656840b4358b3080b42516d380f2dc0f46404c8", "post_mlp_state": "7fd9c911e93175896ad618842c120e77b3f017c17b7eec9c2bfb01fc6dbc8ac2", "pre_attention_input": "de71d4cf2d509a1063f6eac7f50c4885e9a058dee7e4b993ad40f5527f528d00", "pre_mlp_input": "2999a257265024350fd2665e2a8d452a03cd662193b4245d15290408ad29922d" }, "logits_sha256": "873a309a197bfd08fa523edcb440d2697903a34abc02dad1c7f5991509c3c151", "loss_nats": 5.516995429992676, "mixer_summary_sha256": "8450b24bdc777885a3357dc7b8080e7410acbe96943686433da76f379295133a" }, "loss_nats": 5.516995429992676, "loss_scale": 1.0, "mixers": [ { "branch": "attention", "entropy_mean": -0.0, "group": 1, "latest_source_mass": 1.0, "layer": 1, "max_source_mass": 1.0, "mixer_index": 0, "normalized_entropy": 1.0, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 1.0, "median": 1.0, "p05": 1.0, "p95": 1.0 } ], "sources": 1 }, { "branch": "mlp", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 1, "max_source_mass": 0.5, "mixer_index": 1, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 2, "max_source_mass": 0.5, "mixer_index": 2, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 2, "max_source_mass": 0.5, "mixer_index": 3, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 3, "max_source_mass": 0.5, "mixer_index": 4, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 3, "max_source_mass": 0.5, "mixer_index": 5, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 4, "max_source_mass": 0.5, "mixer_index": 6, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 4, "max_source_mass": 0.5, "mixer_index": 7, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6931471824645996, "group": 2, "latest_source_mass": 0.5, "layer": 5, "max_source_mass": 0.5, "mixer_index": 8, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "completed_group_1", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 5, "max_source_mass": 0.3333333432674408, "mixer_index": 9, "normalized_entropy": 1.0000000180537645, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 6, "max_source_mass": 0.3333333432674408, "mixer_index": 10, "normalized_entropy": 1.0000000180537645, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 6, "max_source_mass": 0.3333333432674408, "mixer_index": 11, "normalized_entropy": 1.0000000180537645, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 7, "max_source_mass": 0.3333333432674408, "mixer_index": 12, "normalized_entropy": 1.0000000180537645, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 7, "max_source_mass": 0.3333333432674408, "mixer_index": 13, "normalized_entropy": 1.0000000180537645, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 8, "max_source_mass": 0.3333333432674408, "mixer_index": 14, "normalized_entropy": 1.0000000180537645, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 8, "max_source_mass": 0.3333333432674408, "mixer_index": 15, "normalized_entropy": 1.0000000180537645, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0986123085021973, "group": 3, "latest_source_mass": 0.3333333432674408, "layer": 9, "max_source_mass": 0.3333333432674408, "mixer_index": 16, "normalized_entropy": 1.0000000180537645, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_2", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 9, "max_source_mass": 0.25, "mixer_index": 17, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 10, "max_source_mass": 0.25, "mixer_index": 18, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 10, "max_source_mass": 0.25, "mixer_index": 19, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 11, "max_source_mass": 0.25, "mixer_index": 20, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 11, "max_source_mass": 0.25, "mixer_index": 21, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 12, "max_source_mass": 0.25, "mixer_index": 22, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 12, "max_source_mass": 0.25, "mixer_index": 23, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3862943649291992, "group": 4, "latest_source_mass": 0.25, "layer": 13, "max_source_mass": 0.25, "mixer_index": 24, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_3", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 13, "max_source_mass": 0.20000000298023224, "mixer_index": 25, "normalized_entropy": 1.0000000186840277, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 14, "max_source_mass": 0.20000000298023224, "mixer_index": 26, "normalized_entropy": 1.0000000186840277, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 14, "max_source_mass": 0.20000000298023224, "mixer_index": 27, "normalized_entropy": 1.0000000186840277, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 15, "max_source_mass": 0.20000000298023224, "mixer_index": 28, "normalized_entropy": 1.0000000186840277, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 15, "max_source_mass": 0.20000000298023224, "mixer_index": 29, "normalized_entropy": 1.0000000186840277, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 16, "max_source_mass": 0.20000000298023224, "mixer_index": 30, "normalized_entropy": 1.0000000186840277, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 16, "max_source_mass": 0.20000000298023224, "mixer_index": 31, "normalized_entropy": 1.0000000186840277, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.6094379425048828, "group": 5, "latest_source_mass": 0.20000000298023224, "layer": 17, "max_source_mass": 0.20000000298023224, "mixer_index": 32, "normalized_entropy": 1.0000000186840277, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_4", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 17, "max_source_mass": 0.1666666716337204, "mixer_index": 33, "normalized_entropy": 1.0000001451965654, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 18, "max_source_mass": 0.1666666716337204, "mixer_index": 34, "normalized_entropy": 1.0000001451965654, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 18, "max_source_mass": 0.1666666716337204, "mixer_index": 35, "normalized_entropy": 1.0000001451965654, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 19, "max_source_mass": 0.1666666716337204, "mixer_index": 36, "normalized_entropy": 1.0000001451965654, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 19, "max_source_mass": 0.1666666716337204, "mixer_index": 37, "normalized_entropy": 1.0000001451965654, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 20, "max_source_mass": 0.1666666716337204, "mixer_index": 38, "normalized_entropy": 1.0000001451965654, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 20, "max_source_mass": 0.1666666716337204, "mixer_index": 39, "normalized_entropy": 1.0000001451965654, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.791759729385376, "group": 6, "latest_source_mass": 0.1666666716337204, "layer": 21, "max_source_mass": 0.1666666716337204, "mixer_index": 40, "normalized_entropy": 1.0000001451965654, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_5", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 21, "max_source_mass": 0.1428571492433548, "mixer_index": 41, "normalized_entropy": 1.00000003408302, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 22, "max_source_mass": 0.1428571492433548, "mixer_index": 42, "normalized_entropy": 1.00000003408302, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 22, "max_source_mass": 0.1428571492433548, "mixer_index": 43, "normalized_entropy": 1.00000003408302, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 23, "max_source_mass": 0.1428571492433548, "mixer_index": 44, "normalized_entropy": 1.00000003408302, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 23, "max_source_mass": 0.1428571492433548, "mixer_index": 45, "normalized_entropy": 1.00000003408302, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 24, "max_source_mass": 0.1428571492433548, "mixer_index": 46, "normalized_entropy": 1.00000003408302, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 24, "max_source_mass": 0.1428571492433548, "mixer_index": 47, "normalized_entropy": 1.00000003408302, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9459102153778076, "group": 7, "latest_source_mass": 0.1428571492433548, "layer": 25, "max_source_mass": 0.1428571492433548, "mixer_index": 48, "normalized_entropy": 1.00000003408302, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_6", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 25, "max_source_mass": 0.125, "mixer_index": 49, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 26, "max_source_mass": 0.125, "mixer_index": 50, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 26, "max_source_mass": 0.125, "mixer_index": 51, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 27, "max_source_mass": 0.125, "mixer_index": 52, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 27, "max_source_mass": 0.125, "mixer_index": 53, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 28, "max_source_mass": 0.125, "mixer_index": 54, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 28, "max_source_mass": 0.125, "mixer_index": 55, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.079441547393799, "group": 8, "latest_source_mass": 0.125, "layer": 29, "max_source_mass": 0.125, "mixer_index": 56, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_7", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 29, "max_source_mass": 0.1111111119389534, "mixer_index": 57, "normalized_entropy": 1.0000000180537645, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 30, "max_source_mass": 0.1111111119389534, "mixer_index": 58, "normalized_entropy": 1.0000000180537645, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 30, "max_source_mass": 0.1111111119389534, "mixer_index": 59, "normalized_entropy": 1.0000000180537645, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 31, "max_source_mass": 0.1111111119389534, "mixer_index": 60, "normalized_entropy": 1.0000000180537645, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 31, "max_source_mass": 0.1111111119389534, "mixer_index": 61, "normalized_entropy": 1.0000000180537645, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 32, "max_source_mass": 0.1111111119389534, "mixer_index": 62, "normalized_entropy": 1.0000000180537645, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 32, "max_source_mass": 0.1111111119389534, "mixer_index": 63, "normalized_entropy": 1.0000000180537645, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "output", "entropy_mean": 2.1972246170043945, "group": null, "latest_source_mass": 0.1111111119389534, "layer": null, "max_source_mass": 0.1111111119389534, "mixer_index": null, "normalized_entropy": 1.0000000180537645, "offset": null, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_8", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 } ], "mode": "uniform_group_7_mlp_only", "optimizer_state_after": "b82a349700eda58e2330a22ca2ee1f7fb1406efd47fd9aa306122b368a114192", "optimizer_state_before": "b82a349700eda58e2330a22ca2ee1f7fb1406efd47fd9aa306122b368a114192", "optimizer_state_unchanged": true, "positions": { "attention_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "676759a2d7c0615ad7e1f246e90927fb829d00c8b6c3745032ae6b27ab7d902a", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.0022265110164880753, 0.0021613112185150385, 0.0025601957459002733, 0.0025323182344436646, 0.0027253651060163975, 0.002904455177485943, 0.0029208383057266474, 0.00299624796025455, 0.0030937574338167906, 0.0033804215490818024, 0.0034370317589491606, 0.0038969130255281925, 0.004255065228790045, 0.004119740333408117, 0.003970506135374308, 0.004385598469525576, 0.0041999719105660915, 0.005232699681073427, 0.004664790350943804, 0.004898969549685717, 0.005131533369421959, 0.005069608334451914, 0.0051318020559847355, 0.0058372304774820805, 0.0059690922498703, 0.00554921617731452, 0.0058550527319312096, 0.006255472078919411, 0.006502739619463682, 0.006122888531535864, 0.007071305997669697, 0.006241494789719582 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.00015962477027642308, "first_to_last_ratio": 3.7699040072151497, "imbalance_abs_log_ratio": 1.3270495388586014, "last_quartile_mean": 4.234186599205714e-05, "mean": 8.617327489446325e-05, "normalized": [ 2.5428430303860874, 2.3222064178412967, 2.1011513502014654, 1.8911470095267005, 1.7004447966680725, 1.5353700031893112, 1.4229335846259188, 1.3028620492588128, 1.2049362956920768, 1.1705197150696323, 1.0773612761873523, 1.0133200843661747, 0.9906112963967714, 0.9272530070793641, 0.8550207003106389, 0.8185600560490035, 0.7703816559177782, 0.7173918665339755, 0.6760157258502337, 0.6514535263006792, 0.6184138115448249, 0.6053606047045522, 0.5838821473278404, 0.5697017460729616, 0.5457761369570343, 0.5276086337508958, 0.5104655265174319, 0.4951752820428687, 0.47527333481689216, 0.465789239220051, 0.45896414036514044, 0.45180594922816086 ], "peak_layer": 1, "peak_normalized": 2.5428430303860874, "population_cv": 0.5705341151299013, "rest_mean": 9.280180475562466e-05, "spike_contrast": 0.5428688997682254, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.0379213644191625e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00021912511147093028, 0.00020011213200632483, 0.00018106309289578348, 0.00016296633111778647, 0.00014653289690613747, 0.00013230786134954542, 0.0001226188469445333, 0.00011227188952034339, 0.0001038333066389896, 0.00010086751717608422, 9.283974941354245e-05, 8.732111018616706e-05, 8.536421955795959e-05, 7.990442827576771e-05, 7.367993384832516e-05, 7.053800072753802e-05, 6.63863102090545e-05, 6.182000652188435e-05, 5.825448897667229e-05, 5.6137883802875876e-05, 5.329074338078499e-05, 5.216590579948388e-05, 5.031503678765148e-05, 4.909306517220102e-05, 4.703131708083674e-05, 4.5465763832908124e-05, 4.3988486140733585e-05, 4.2670875700423494e-05, 4.095585973118432e-05, 4.013858415419236e-05, 3.955044303438626e-05, 3.893359826179221e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.00021442661636683624, "first_to_last_ratio": 1.843144421770586, "imbalance_abs_log_ratio": 0.6114730379455638, "last_quartile_mean": 0.0001163373926829081, "mean": 0.00015221274247778638, "normalized": [ 1.7741446897870732, 1.648461275666186, 1.5301464224684427, 1.4217121808823485, 1.3265065496562456, 1.2454420544341354, 1.1897998831173078, 1.1336248235930768, 1.0869089599614004, 1.0643003125599004, 1.0227030055526172, 0.9941765648180964, 0.9788958472869692, 0.9511102301475627, 0.9228060651723192, 0.9052418824754537, 0.8846484396889256, 0.865292475366091, 0.8497311645646503, 0.8388691935254817, 0.8256303537004397, 0.8176325372729673, 0.8076852907347736, 0.8000670639426607, 0.7892588692968658, 0.7814415503461742, 0.7731202151361662, 0.7664994567889658, 0.758490072461763, 0.7529625284402992, 0.7485174912671357, 0.7441725498875049 ], "peak_layer": 1, "peak_normalized": 1.7741446897870732, "population_cv": 0.27364872432853743, "rest_mean": 0.00015762320577373935, "spike_contrast": 0.7803181014868814, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012299624067964033, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00027004742878489196, 0.0002509168116375804, 0.0002329077833564952, 0.00021640271006617695, 0.00020191119983792305, 0.00018957215070258826, 0.0001811027032090351, 0.00017255214333999902, 0.00016544139361940324, 0.00016200006939470768, 0.00015566842921543866, 0.0001513263414381072, 0.00014900042151566595, 0.0001447710965294391, 0.00014046284195501357, 0.0001377893495373428, 0.00013465476513374597, 0.00013170854072086513, 0.00012933991092722863, 0.0001276865805266425, 0.00012567146040964872, 0.00012445409083738923, 0.00012293999316170812, 0.00012178040196886286, 0.00012013525702059269, 0.00011894536146428436, 0.00011767874821089208, 0.00011667098442558199, 0.00011545185407157987, 0.00011461049143690616, 0.00011393390013836324, 0.00011327254469506443 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.19015566632151604, "first_to_last_ratio": 1.8431444167877922, "imbalance_abs_log_ratio": 0.6114730352421437, "last_quartile_mean": 0.10316916275769472, "mean": 0.13498378219082952, "normalized": [ 1.7741447141792424, 1.6484611693507762, 1.5301464164964773, 1.421712099711509, 1.3265065258282553, 1.245442086656075, 1.1897999853143832, 1.1336247998774232, 1.0869090145314644, 1.0643003539468234, 1.0227030148964142, 0.9941765578539046, 0.978895844117059, 0.9511102293531465, 0.9228060470726628, 0.9052419251832852, 0.8846484748200404, 0.8652925213936268, 0.8497311905993837, 0.8388692019864461, 0.8256303029992552, 0.8176324960789627, 0.8076852727084025, 0.800067049482784, 0.789258877083647, 0.7814415614650243, 0.7731201949085273, 0.7664994759042474, 0.758490077799444, 0.7529625178867759, 0.7485174642151933, 0.7441725362993381 ], "peak_layer": 1, "peak_normalized": 1.7741447141792424, "population_cv": 0.2736487184871982, "rest_mean": 0.13978183573042904, "spike_contrast": 0.7803180757143806, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.10907429307699204, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.23948076367378235, 0.2225155234336853, 0.20654495060443878, 0.1919080764055252, 0.17905686795711517, 0.16811448335647583, 0.16060370206832886, 0.1530209630727768, 0.14671508967876434, 0.14366328716278076, 0.13804832100868225, 0.13419771194458008, 0.1321350634098053, 0.12838445603847504, 0.12456385046243668, 0.12219297885894775, 0.11941319704055786, 0.116800457239151, 0.11469992995262146, 0.11323373764753342, 0.11144670099020004, 0.11036712676286697, 0.10902441293001175, 0.10799607634544373, 0.10653714835643768, 0.10548193752765656, 0.10435868799686432, 0.1034649983048439, 0.10238385945558548, 0.10163772851228714, 0.10103771835565567, 0.10045122355222702 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 6.883365585963475e-05, "first_to_last_ratio": 2.2497546817252747, "imbalance_abs_log_ratio": 0.8108211799277664, "last_quartile_mean": 3.059607183786284e-05, "mean": 4.691841348858361e-05, "normalized": [ 1.7359303646705315, 1.6475859007944922, 1.5536416135247637, 1.4921074478374423, 1.436565604934822, 1.343834173349448, 1.2787439414688093, 1.2483315192759759, 1.191227587726315, 1.1603086071531024, 1.0995306735006558, 1.059977946939621, 1.0583984121521266, 1.0176934649371145, 0.9754909410255406, 0.9631621800279858, 0.9242301483750994, 0.8752327792179452, 0.8454160845678106, 0.8232909667825862, 0.7863453139125789, 0.77746685672799, 0.7497386589361914, 0.7388508716017261, 0.7093180450413769, 0.6908354487677191, 0.6685471901846501, 0.6592354484095886, 0.631440334976849, 0.6235901145414114, 0.6199506557967245, 0.6139807028410055 ], "peak_layer": 1, "peak_normalized": 1.7359303646705315, "population_cv": 0.3220412433027962, "rest_mean": 4.907019663530971e-05, "spike_contrast": 0.7193528234378936, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.529878449626267e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 8.144709863699973e-05, 7.730211655143648e-05, 7.289439963642508e-05, 7.000731420703232e-05, 6.740137905580923e-05, 6.305056740529835e-05, 5.999663699185476e-05, 5.856973439222202e-05, 5.5890508519951254e-05, 5.4439839004771784e-05, 5.1588234782684594e-05, 4.9732483603293076e-05, 4.965837433701381e-05, 4.774856279254891e-05, 4.576848732540384e-05, 4.519004141911864e-05, 4.3363412260077894e-05, 4.106453343410976e-05, 3.966558142565191e-05, 3.862750600092113e-05, 3.6894074582960457e-05, 3.647751145763323e-05, 3.517654840834439e-05, 3.4665710700210184e-05, 3.328007733216509e-05, 3.241290323785506e-05, 3.136717350571416e-05, 3.0930281354812905e-05, 2.9626178729813546e-05, 2.925785884144716e-05, 2.9087101211189292e-05, 2.8807000489905477e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 0.0001734125216898974, "first_to_last_ratio": 1.495517223041696, "imbalance_abs_log_ratio": 0.4024721155990803, "last_quartile_mean": 0.00011595488103921525, "mean": 0.00013990532806928968, "normalized": [ 1.3768889945483427, 1.3340940553978193, 1.2910049686488183, 1.2527203273458163, 1.2147536525239127, 1.1769854337196524, 1.14744939519443, 1.1220955988146075, 1.0944022472959734, 1.0773022658134643, 1.0508559212107247, 1.03206209473486, 1.0208171898483585, 1.000561882500425, 0.9805161605170362, 0.9663129296065233, 0.9487424991891081, 0.931931984738623, 0.9181974345490227, 0.9075104531838204, 0.8943640941785878, 0.8862664021756573, 0.8757830774574916, 0.86790401991425, 0.8562894022841879, 0.8478155487572495, 0.8387227683262297, 0.8314011142413483, 0.8224071457687988, 0.8161875050523302, 0.8112199678685613, 0.8064334645939688 ], "peak_layer": 1, "peak_normalized": 1.3768889945483427, "population_cv": 0.16325957486062354, "rest_mean": 0.0001431148236776978, "spike_contrast": 0.8564734849544942, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001225740517838858, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0001926341064972803, 0.00018664686649572104, 0.00018061847367789596, 0.00017526224837638438, 0.0001699505082797259, 0.00016466653323732316, 0.00016053428407758474, 0.00015698715287726372, 0.00015311270544771105, 0.00015072032692842185, 0.00014702034241054207, 0.00014439098595175892, 0.00014281776384450495, 0.00013998393842484802, 0.00013717943511437625, 0.00013519232743419707, 0.00013273413060232997, 0.00013038225006312132, 0.00012846071331296116, 0.00012696554767899215, 0.0001251263020094484, 0.00012399339175317436, 0.0001225267187692225, 0.00012142439663875848, 0.00011979944974882528, 0.00011861391249112785, 0.00011734178406186402, 0.00011631744564510882, 0.00011505914153531194, 0.00011418898066040128, 0.00011349399574100971, 0.00011282433843007311 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 0.00014675117381557357, "first_to_last_ratio": 1.2676837465168667, "imbalance_abs_log_ratio": 0.23719141364257146, "last_quartile_mean": 0.00011576323686313117, "mean": 0.00013042020850662084, "normalized": [ 1.169988069421413, 1.156175152925158, 1.1417665260225645, 1.1306726331060184, 1.1183621029407895, 1.1062694829504214, 1.0940115056230622, 1.0844989956600726, 1.0711591662817939, 1.0579765491085218, 1.0443900226505625, 1.035471661013805, 1.0260498632741444, 1.013992390086173, 1.0062479316494695, 0.9977001177294099, 0.9863163473759529, 0.9769959732660659, 0.9687585669786032, 0.9602628595878113, 0.9503845997255937, 0.9426104617668664, 0.9330243108845692, 0.9259760938667744, 0.914797625470175, 0.906289421438531, 0.8956357003805738, 0.888470885240542, 0.8806216558595509, 0.8756991509238615, 0.8712858841028995, 0.8681382926882502 ], "peak_layer": 1, "peak_normalized": 1.169988069421413, "population_cv": 0.09123006364857297, "rest_mean": 0.00013202972385678786, "spike_contrast": 0.9219804606102014, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012172882561571896, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00015259008796419948, 0.00015078860451467335, 0.00014890942838974297, 0.0001474625605624169, 0.00014585701865144074, 0.00014427989663090557, 0.00014268120867200196, 0.00014144058513920754, 0.00013970080181024969, 0.0001379815221298486, 0.00013620956451632082, 0.00013504642993211746, 0.0001338176371064037, 0.00013224509893916547, 0.00013123506505507976, 0.0001301202573813498, 0.00012863558367826045, 0.00012742001854348928, 0.00012634569429792464, 0.00012523768236860633, 0.00012394935765769333, 0.00012293545296415687, 0.00012168522516731173, 0.00012076599523425102, 0.00011930809705518186, 0.00011819845531135798, 0.00011680899478960782, 0.00011587455810513347, 0.00011485085997264832, 0.00011420886585256085, 0.00011363328667357564, 0.00011322277714498341 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00030002388848515693, "first_to_last_ratio": 2.2836575149168885, "imbalance_abs_log_ratio": 0.8257783308640597, "last_quartile_mean": 0.0001313786706305109, "mean": 0.0001947244400071213, "normalized": [ 1.8998833386658438, 1.7835645331795598, 1.6713073472446418, 1.5703972875779748, 1.4780250735507643, 1.3761669788219328, 1.3066447563403325, 1.2401013786965793, 1.1762106459027857, 1.150547637187856, 1.0768735950666006, 1.0264900078881174, 1.0140784159549234, 0.9666871251601796, 0.9096841069789114, 0.8754807117769676, 0.8421876125778792, 0.8073809902734768, 0.77911667677541, 0.7607651063686129, 0.7403822030704592, 0.7308855611321243, 0.7152791523187013, 0.7043381880055872, 0.6944157299089508, 0.686864629891504, 0.680599049276497, 0.6764772714226681, 0.6702412094778135, 0.6672258960044359, 0.6635863560708457, 0.6581114274310631 ], "peak_layer": 1, "peak_normalized": 1.8998833386658438, "population_cv": 0.3592853832280775, "rest_mean": 0.00020492727344390005, "spike_contrast": 0.6813594750078023, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013962913944851608, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0003699537192005664, 0.0003473036049399525, 0.0003254443872720003, 0.00030579473241232336, 0.00028780760476365685, 0.00026797334430739284, 0.0002544356684666127, 0.0002414780465187505, 0.0002290369593538344, 0.00022403974435292184, 0.00020969360775779933, 0.00019988269195891917, 0.0001974658516701311, 0.00018823760910890996, 0.00017713772831484675, 0.00017047749133780599, 0.00016399451124016196, 0.0001572168112033978, 0.00015171305858530104, 0.00014813955931458622, 0.00014417050988413393, 0.00014232128160074353, 0.00013928233238402754, 0.0001371518592350185, 0.00013521971413865685, 0.00013374933041632175, 0.00013252926873974502, 0.00013172665785532445, 0.00013051234418526292, 0.00012992518895771354, 0.00012921648158226162, 0.000128150379168801 ] } } }, "mlp_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "afe29f3ef4d2992353f0bb59fe7c5c2c5050fca671a53184825a8d63cd83a2bc", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.0027073738165199757, 0.002722575329244137, 0.0026850115973502398, 0.0026861224323511124, 0.00271838391199708, 0.0026645169127732515, 0.0026859452482312918, 0.0027359756641089916, 0.0026767742820084095, 0.0027451682835817337, 0.0026999011170119047, 0.002663193503394723, 0.002626648638397455, 0.0026531771291047335, 0.0026517712976783514, 0.002676903735846281, 0.002670086920261383, 0.0026355870068073273, 0.0026610493659973145, 0.0026388936676084995, 0.0026469926815479994, 0.00264862016774714, 0.002641494618728757, 0.002674198243767023, 0.002557571977376938, 0.00252733426168561, 0.0025468147359788418, 0.0026856842450797558, 0.002551089273765683, 0.0025778315030038357, 0.0026001501828432083, 0.0025662274565547705 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.00015858857113926206, "first_to_last_ratio": 3.757867690304466, "imbalance_abs_log_ratio": 1.3238516928952826, "last_quartile_mean": 4.220174423608114e-05, "mean": 8.573221236929385e-05, "normalized": [ 2.5461641896458365, 2.3032502265274117, 2.0924715218207024, 1.8886722264368014, 1.7031663195050915, 1.537434754996766, 1.4223955235407997, 1.3049501874408618, 1.2055297161045848, 1.1692542365736316, 1.0769571962730058, 1.0144505847411098, 0.9913601784485884, 0.9317001386730132, 0.857757082878185, 0.8155634736467603, 0.7682612945963089, 0.7217806418748633, 0.678338533281248, 0.6511820384364709, 0.6192882306865346, 0.6053793146733001, 0.5848721334223365, 0.5718144052517429, 0.5462281883927541, 0.527515083157495, 0.5112993585545178, 0.496920907994256, 0.47722358366421574, 0.46552107689700334, 0.4601234027970099, 0.45317424906679377 ], "peak_layer": 1, "peak_normalized": 2.5461641896458365, "population_cv": 0.5686091737198813, "rest_mean": 9.231269224848668e-05, "spike_contrast": 0.5437781067692293, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.0197621021652594e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00021828828903380781, 0.00019746273756027222, 0.00017939221288543195, 0.0001619200484128669, 0.00014601621660403907, 0.000131807682919316, 0.00012194511509733275, 0.00011187626660102978, 0.00010335272963857278, 0.00010024275252362713, 9.232992306351662e-05, 8.697109296917915e-05, 8.499150135321543e-05, 7.987671415321529e-05, 7.353741239057854e-05, 6.992006092332304e-05, 6.586474046343938e-05, 6.187985127326101e-05, 5.8155463193543255e-05, 5.582727681030519e-05, 5.309295011102222e-05, 5.1900507969548926e-05, 5.0142381951445714e-05, 4.902291402686387e-05, 4.682935104938224e-05, 4.522503513726406e-05, 4.3834825191879645e-05, 4.260212881490588e-05, 4.0913433622336015e-05, 3.991015182691626e-05, 3.944739728467539e-05, 3.885163096128963e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.00021242323236947414, "first_to_last_ratio": 1.8345651540958352, "imbalance_abs_log_ratio": 0.6068074801791611, "last_quartile_mean": 0.0001157894184871111, "mean": 0.00015111673860701558, "normalized": [ 1.771707915810251, 1.6366668487510905, 1.5242909365839221, 1.4186705636795862, 1.3264449439106374, 1.2453857912414694, 1.1893415085703798, 1.1330085309323608, 1.0858463851390703, 1.062861239127963, 1.023237806768965, 0.9942126959777651, 0.978436257513136, 0.9531027445378829, 0.9232414968561398, 0.904013237249231, 0.8853704564278231, 0.8680164030054869, 0.8515402788738432, 0.8395817783841374, 0.8267435189825344, 0.8185143642831039, 0.809172703595369, 0.8007918347797284, 0.7909896391756025, 0.7821052393452608, 0.7753388667989797, 0.7682453290925442, 0.7604995795745633, 0.7549901567959042, 0.7508572350862155, 0.7467737131490535 ], "peak_layer": 1, "peak_normalized": 1.771707915810251, "population_cv": 0.2714170004285523, "rest_mean": 0.00015645500981783562, "spike_contrast": 0.7816309251520459, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012229007406858728, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0002677347220014781, 0.00024732775636948645, 0.00023034587502479553, 0.00021438486874103546, 0.00020044803386554122, 0.0001881986390799284, 0.00017972940986510366, 0.00017121655400842428, 0.00016408956435043365, 0.00016061612404882908, 0.0001546283601783216, 0.00015024218009784818, 0.00014785809617023915, 0.0001440297783119604, 0.00013951724395155907, 0.000136611532070674, 0.0001337942958343774, 0.00013117180787958205, 0.0001286819897359237, 0.00012687486014328897, 0.00012493478425312787, 0.00012369122123345733, 0.00012227953993715346, 0.00012101305037504062, 0.00011953177454415709, 0.00011818919301731512, 0.00011716668086592108, 0.00011609472858253866, 0.00011492421617731452, 0.00011409165017539635, 0.00011346709652571008, 0.00011285000800853595 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.18837904930114746, "first_to_last_ratio": 1.8345651581276443, "imbalance_abs_log_ratio": 0.6068074823768531, "last_quartile_mean": 0.10268321540206671, "mean": 0.13401183625683188, "normalized": [ 1.7717078885231814, 1.6366668119471057, 1.524290957183596, 1.418670514273678, 1.326444920903592, 1.2453858718827995, 1.1893415500975306, 1.1330085716119305, 1.0858463365841577, 1.062861212873729, 1.0232377434953237, 0.9942127340740732, 0.9784362443707608, 0.9531027271661117, 0.9232414917376033, 0.904013241653558, 0.885370477646398, 0.8680164390463408, 0.8515402681847718, 0.8395816969955481, 0.8267435345782925, 0.8185144279720658, 0.8091727254708025, 0.8007918405918669, 0.7909896328038464, 0.7821052665924377, 0.7753388466443417, 0.7682453528580236, 0.7604996016741158, 0.7549901615676585, 0.7508572336421532, 0.7467736753526057 ], "peak_layer": 1, "peak_normalized": 1.7717078885231814, "population_cv": 0.27141699472708714, "rest_mean": 0.1387458680956452, "spike_contrast": 0.7816309473985974, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.10844806432724, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.23742982745170593, 0.2193327248096466, 0.2042730301618576, 0.19011864066123962, 0.1777593195438385, 0.16689644753932953, 0.15938584506511688, 0.151836559176445, 0.1455162614583969, 0.1424359828233719, 0.13712596893310547, 0.13323627412319183, 0.13112203776836395, 0.12772704660892487, 0.1237252876162529, 0.1211484745144844, 0.11865012347698212, 0.11632447689771652, 0.11411647498607635, 0.11251388490200043, 0.11079341918230057, 0.10969062149524689, 0.10843872278928757, 0.10731558501720428, 0.10600197315216064, 0.10481136292219162, 0.10390458256006241, 0.1029539704322815, 0.10191594809293747, 0.10117761790752411, 0.10062375664710999, 0.10007651150226593 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 6.873187476230669e-05, "first_to_last_ratio": 2.248757500145603, "imbalance_abs_log_ratio": 0.8103778415280068, "last_quartile_mean": 3.0564378221242805e-05, "mean": 4.685761763312257e-05, "normalized": [ 1.7400006587426948, 1.6405194725375354, 1.5551306973978087, 1.4944078341288562, 1.4327724019957897, 1.3414187773842097, 1.2804088050749736, 1.2499327782691996, 1.1927955171655855, 1.1608931065609767, 1.0981756972550076, 1.0577423904261343, 1.0542786038595535, 1.0218640087393278, 0.9775807389332813, 0.9570972402588515, 0.9209982861589985, 0.8815092249082567, 0.8478232134430647, 0.8204952153707635, 0.7871780668180614, 0.7759097745574741, 0.7506963524238334, 0.7421155343193898, 0.7072927325438048, 0.6891814936654451, 0.6695399899996352, 0.6616343983211914, 0.6345111343049281, 0.6212687927046295, 0.6200231913288072, 0.6148038704019305 ], "peak_layer": 1, "peak_normalized": 1.7400006587426948, "population_cv": 0.3217536161653951, "rest_mean": 4.900405669845727e-05, "spike_contrast": 0.719671983430414, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.526684668031521e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 8.153228554874659e-05, 7.687083416385576e-05, 7.286971958819777e-05, 7.002439087955281e-05, 6.71363013680093e-05, 6.285568815656006e-05, 5.999690620228648e-05, 5.8568872191244736e-05, 5.5891556257847697e-05, 5.439668530016206e-05, 5.145789691596292e-05, 4.9563288484932855e-05, 4.9400983698433265e-05, 4.7882112994557247e-05, 4.580710447044112e-05, 4.484729652176611e-05, 4.3155785533599555e-05, 4.130542220082134e-05, 3.972697595600039e-05, 3.844645107164979e-05, 3.688528886414133e-05, 3.635728353401646e-05, 3.517584264045581e-05, 3.477376594673842e-05, 3.314205241622403e-05, 3.229340290999971e-05, 3.1373048841487616e-05, 3.10026116494555e-05, 2.9731680115219206e-05, 2.9111175535945222e-05, 2.9052809622953646e-05, 2.8808244678657502e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 0.0001719075116852764, "first_to_last_ratio": 1.4894635014227988, "imbalance_abs_log_ratio": 0.39841598895878805, "last_quartile_mean": 0.00011541572621354135, "mean": 0.00013894635094402474, "normalized": [ 1.3738985779819257, 1.3296082759737424, 1.2883384004275333, 1.2499940655466777, 1.2130730176151365, 1.1755503979222366, 1.147409009170091, 1.11990595257071, 1.0929037170303153, 1.075598370036652, 1.0512762916941902, 1.0312594730951, 1.0194418974669357, 1.0018996501730997, 0.9803460168062212, 0.9648004557293595, 0.9493010807852901, 0.9344788929945884, 0.9198953859218769, 0.908001147597105, 0.8952913731815082, 0.886953676542665, 0.8771894463102521, 0.8683888934041172, 0.8579399877904575, 0.8482799643655919, 0.8408636339472488, 0.8330870307470335, 0.82434015220157, 0.8181187963300758, 0.8135579418994486, 0.809009026741244 ], "peak_layer": 1, "peak_normalized": 1.3738985779819257, "population_cv": 0.16161423472023242, "rest_mean": 0.0001421073115866252, "spike_contrast": 0.8576417505420824, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012187716347398237, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0001908981939777732, 0.0001847442181315273, 0.00017900991952046752, 0.00017368211410939693, 0.00016855206922627985, 0.00016333843814209104, 0.00015942829486448318, 0.00015560684551019222, 0.0001518549834145233, 0.00014945046859793365, 0.00014607100456487387, 0.0001432897406630218, 0.00014164773165248334, 0.00013921030040364712, 0.000136215501697734, 0.0001340555027127266, 0.0001319019211223349, 0.00012984243221580982, 0.0001278161071240902, 0.00012616344611160457, 0.00012439746933523566, 0.00012323897681199014, 0.00012188227265141904, 0.00012065946793882176, 0.00011920763063244522, 0.00011786540562752634, 0.00011683493357850239, 0.00011575440294109285, 0.0001145390560850501, 0.00011367462138878182, 0.00011304090730845928, 0.00011240885214647278 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 0.00014552221000485588, "first_to_last_ratio": 1.262697055715921, "imbalance_abs_log_ratio": 0.2332499537229619, "last_quartile_mean": 0.00011524712863320019, "mean": 0.00012953460168318998, "normalized": [ 1.1676213631634658, 1.1525055688027706, 1.1391580058081985, 1.1289307982409096, 1.1166731577821778, 1.1054001772932092, 1.094813144753513, 1.0822854389705017, 1.0679308603491235, 1.0564894822224282, 1.0456838360716472, 1.0337072702557388, 1.0234945545465333, 1.0158819477934218, 1.0051772952892828, 0.9951467950359763, 0.9866643397779161, 0.9787559418439278, 0.970448736962688, 0.9607908684400951, 0.9509939230158972, 0.9428061355388626, 0.934924980051271, 0.9261034265480621, 0.9170582603044295, 0.9058562786860701, 0.8981778407016658, 0.8907564927732574, 0.8831168120837269, 0.8778216107141017, 0.8745192645108896, 0.8703053916682415 ], "peak_layer": 1, "peak_normalized": 1.1676213631634658, "population_cv": 0.08967236486753827, "rest_mean": 0.00013110875065817877, "spike_contrast": 0.9231588022206524, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012103419721825049, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00015124736819416285, 0.00014928934979252517, 0.000147560378536582, 0.00014623560127802193, 0.00014464781270362437, 0.00014318757166620344, 0.00014181618462316692, 0.00014019341324456036, 0.0001383339986205101, 0.00013685194426216185, 0.00013545223919209093, 0.00013390085950959474, 0.00013257795944809914, 0.00013159186346456409, 0.00013020524056628346, 0.00012890594371128827, 0.00012780717224813998, 0.00012678276107180864, 0.0001257066905964166, 0.0001244556624442339, 0.00012318661902099848, 0.0001221260172314942, 0.00012110513489460573, 0.0001199624384753406, 0.00011879077646881342, 0.00011733973224181682, 0.00011634510883595794, 0.00011538378748809919, 0.0001143941844929941, 0.00011370827269274741, 0.00011328050459269434, 0.0001127346622524783 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00029741139769612346, "first_to_last_ratio": 2.278072415841521, "imbalance_abs_log_ratio": 0.8233296537390045, "last_quartile_mean": 0.00013055396993877366, "mean": 0.00019328556300024502, "normalized": [ 1.889398576482247, 1.780233894901915, 1.6772594716902607, 1.566224958959574, 1.4762232610465842, 1.375571144634086, 1.3083686481730985, 1.2364401527267181, 1.173148897934763, 1.1508864184295666, 1.080457040307468, 1.0318582947152708, 1.0081957004611959, 0.9698994715426978, 0.9066127339785177, 0.8754550048258108, 0.8446146862114313, 0.8115724449181009, 0.7806677558079268, 0.7613480996239201, 0.7403134026279838, 0.7294052762839086, 0.7160186976339193, 0.7062575707090604, 0.6942777320753531, 0.6866005528973866, 0.6829938476675228, 0.6753403086482458, 0.6715967314167978, 0.6686860556999406, 0.6640286432895989, 0.6600445236791291 ], "peak_layer": 1, "peak_normalized": 1.889398576482247, "population_cv": 0.3579154260008552, "rest_mean": 0.00020340604748559632, "spike_contrast": 0.681567477924495, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013863494677934796, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0003651934675872326, 0.0003440935106482357, 0.00032419004128314555, 0.0003027286729775369, 0.0002853326441254467, 0.00026587804313749075, 0.00025288877077400684, 0.00023898603103589267, 0.00022675274522043765, 0.00022244972933549434, 0.0002088367473334074, 0.00019944331143051386, 0.00019486967357806861, 0.00018746756541077048, 0.00017523515271022916, 0.00016921281348913908, 0.0001632518251426518, 0.00015686523693148047, 0.00015089180669747293, 0.00014715759607497603, 0.00014309189282357693, 0.00014098350948188454, 0.00013839607709087431, 0.0001365093921776861, 0.0001341938623227179, 0.00013270997442305088, 0.00013201285037212074, 0.00013053353177383542, 0.0001298099523410201, 0.00012924736074637622, 0.00012834715016651899, 0.00012757707736454904 ] } } }, "post_attention_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "31853ffb07b24cc59e774bb20656840b4358b3080b42516d380f2dc0f46404c8", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.0022265110164880753, 0.0040087271481752396, 0.005379301495850086, 0.006410556845366955, 0.0027253651060163975, 0.004838948138058186, 0.006175771821290255, 0.00742313964292407, 0.0030937574338167906, 0.005150026176124811, 0.007019025273621082, 0.008143599145114422, 0.004255065228790045, 0.0066224560141563416, 0.008129785768687725, 0.010099578648805618, 0.0041999719105660915, 0.00719039561226964, 0.009030314162373543, 0.010736005380749702, 0.005131533369421959, 0.00787449348717928, 0.009912566281855106, 0.012113703414797783, 0.0059690922498703, 0.00876442901790142, 0.010806876234710217, 0.012353328987956047, 0.006502739619463682, 0.009470963850617409, 0.012123330496251583, 0.01384514570236206 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.00015962236011546338, "first_to_last_ratio": 3.7698422272252903, "imbalance_abs_log_ratio": 1.3270331510418396, "last_quartile_mean": 4.2341920561739244e-05, "mean": 8.61727822893954e-05, "normalized": [ 2.542734798705971, 2.3221448837042376, 2.101145461274622, 1.8911858524935854, 1.7004204056817789, 1.5354142426015858, 1.4229001770142031, 1.3028733810328659, 1.2049412416934622, 1.1705258152816296, 1.0773908232649334, 1.0133220774376726, 0.990609360100461, 0.9272534949358001, 0.8550385909364457, 0.8185651575045715, 0.7703972051466339, 0.7173947009730917, 0.6760256695667757, 0.6514523109028573, 0.618419077603279, 0.6053620388084041, 0.5838930841859182, 0.5697043695032278, 0.5457802700901755, 0.5276149005446632, 0.5104701332743111, 0.49517988582787786, 0.47527562953524777, 0.46579059316028215, 0.45896435764104826, 0.45181000957238127 ], "peak_layer": 1, "peak_normalized": 2.542734798705971, "population_cv": 0.5705202801130907, "rest_mean": 9.280119640472414e-05, "spike_contrast": 0.542873885449774, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.03793460666202e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00021911453222855926, 0.00020010568550787866, 0.00018106155039276928, 0.00016296874673571438, 0.00014652995741926134, 0.00013231091725174338, 0.0001226152671733871, 0.00011227222421439365, 0.00010383313929196447, 0.00010086746624438092, 9.284176485380158e-05, 8.732078276807442e-05, 8.536356472177431e-05, 7.99040135461837e-05, 7.368105434579775e-05, 7.053803710732609e-05, 6.638727063545957e-05, 6.181989738252014e-05, 5.825501284562051e-05, 5.613745815935545e-05, 5.329089253791608e-05, 5.216573117650114e-05, 5.0315691623836756e-05, 4.9093010602518916e-05, 4.703140439232811e-05, 4.546604395727627e-05, 4.398863165988587e-05, 4.267102849553339e-05, 4.095582335139625e-05, 4.013847137684934e-05, 3.955023566959426e-05, 3.893372559105046e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.00021442477373057045, "first_to_last_ratio": 1.8431254562763404, "imbalance_abs_log_ratio": 0.6114627481432037, "last_quartile_mean": 0.00011633759004325839, "mean": 0.00015221248349917005, "normalized": [ 1.7740805953138137, 1.6484457246979003, 1.5301434809496548, 1.4217241600875643, 1.3264958046529274, 1.2454577490392524, 1.1897861330408546, 1.1336265611690741, 1.086914920172498, 1.064301358569379, 1.0227209980571423, 0.9941753882589759, 0.9788979908211966, 0.9511107967643203, 0.9228143274453928, 0.9052447611165039, 0.8846530997418666, 0.8652930871753776, 0.8497381552743479, 0.8388656494639406, 0.8256334792987466, 0.8176309647342147, 0.8076951735882674, 0.8000661307359944, 0.7892605467734257, 0.78144613040264, 0.7731219607566042, 0.7664988488799842, 0.7584935618388989, 0.7529639051546407, 0.7485195296370439, 0.7441790263875565 ], "peak_layer": 1, "peak_normalized": 1.7740805953138137, "population_cv": 0.2736411578809574, "rest_mean": 0.00015762286892384981, "spike_contrast": 0.7803207938393821, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012299640220589936, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00027003721334040165, 0.00025091401766985655, 0.00023290693934541196, 0.0002164041652576998, 0.000201909220777452, 0.0001895742170745507, 0.00018110030214302242, 0.00017255211423616856, 0.00016544201935175806, 0.00016199995297938585, 0.00015567090304102749, 0.00015132590488065034, 0.0001490004942752421, 0.00014477093645837158, 0.00014046386058907956, 0.00013778955326415598, 0.0001346552453469485, 0.00013170840975362808, 0.00012934075493831187, 0.00012768582382705063, 0.00012567172234412283, 0.00012445363972801715, 0.00012294128828216344, 0.00012178005272289738, 0.00012013530795229599, 0.00011894585622940212, 0.00011767881369451061, 0.00011667069338727742, 0.00011545218876563013, 0.00011461050598882139, 0.00011393401655368507, 0.00011327333777444437 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.190154030919075, "first_to_last_ratio": 1.84312545379361, "imbalance_abs_log_ratio": 0.6114627467961817, "last_quartile_mean": 0.10316933691501617, "mean": 0.13498355145566165, "normalized": [ 1.774080628234443, 1.6484456620184311, 1.5301434020504485, 1.4217242444591272, 1.326495766996536, 1.2454576834375484, 1.189786122599316, 1.133626627255238, 1.0869149569673349, 1.0643014004698326, 1.0227209907477024, 0.9941753870542451, 0.9788979589698585, 0.9511107512164827, 0.9228143032197887, 0.9052447972745712, 0.8846531331847195, 0.8652931173486506, 0.8497382179112226, 0.8388656682549475, 0.8256334805754935, 0.8176309131099365, 0.8076951535479161, 0.8000661540388709, 0.7892605573866218, 0.7814461538045975, 0.7731219580171443, 0.7664988542586116, 0.7584935821787124, 0.7529638601652359, 0.7485195164462104, 0.7441789968002049 ], "peak_layer": 1, "peak_normalized": 1.774080628234443, "population_cv": 0.27364115345546325, "rest_mean": 0.13978153549962574, "spike_contrast": 0.7803207857775153, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.10907443761825561, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.2394717037677765, 0.22251304984092712, 0.2065441906452179, 0.19190938770771027, 0.1790551096200943, 0.16811630129814148, 0.16060155630111694, 0.1530209481716156, 0.14671564102172852, 0.1436631828546524, 0.13805051147937775, 0.13419732451438904, 0.13213512301445007, 0.1283843070268631, 0.1245647519826889, 0.12219315767288208, 0.11941362172365189, 0.11680033802986145, 0.11470068246126175, 0.1132330670952797, 0.11144693940877914, 0.11036672443151474, 0.10902556031942368, 0.10799577087163925, 0.10653719305992126, 0.10548237711191177, 0.1043587476015091, 0.10346473753452301, 0.10238415747880936, 0.10163773596286774, 0.10103782266378403, 0.10045192390680313 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 6.883367768750759e-05, "first_to_last_ratio": 2.249759357551583, "imbalance_abs_log_ratio": 0.810823258297238, "last_quartile_mean": 3.059601795030176e-05, "mean": 4.691846504556452e-05, "normalized": [ 1.7359059710084281, 1.6475789727938472, 1.5536456441204052, 1.4921194549590373, 1.4365657321885223, 1.3438513058553838, 1.2787343947807137, 1.2483299149152907, 1.1912354282513808, 1.1603083401306369, 1.099547144002948, 1.059972440022979, 1.0583955432739494, 1.0176800955770247, 0.9754963047746087, 0.9631595708780456, 0.9242400656751482, 0.8752301116137658, 0.8454245377074081, 0.8232853322610043, 0.7863494898171388, 0.7774642190168813, 0.7497486129019063, 0.7388483538626245, 0.709315327139865, 0.6908343794807076, 0.6685469983108738, 0.6592365073812129, 0.6314341746588822, 0.6235898169926835, 0.6199474545603838, 0.6139783610863121 ], "peak_layer": 1, "peak_normalized": 1.7359059710084281, "population_cv": 0.3220406003070692, "rest_mean": 4.9070239684725595e-05, "spike_contrast": 0.7193541792517962, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.52988819940947e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 8.144604362314567e-05, 7.730187644483522e-05, 7.28946688468568e-05, 7.000795449130237e-05, 6.740145909134299e-05, 6.305144052021205e-05, 5.999625500408001e-05, 5.8569723478285596e-05, 5.589093780145049e-05, 5.4439886298496276e-05, 5.158906424185261e-05, 4.973227987647988e-05, 4.9658294301480055e-05, 4.774798799189739e-05, 4.576878927764483e-05, 4.51899686595425e-05, 4.3363925215089694e-05, 4.1064453398576006e-05, 3.9666021621087566e-05, 3.86272840842139e-05, 3.689431105158292e-05, 3.6477427784120664e-05, 3.5177054087398574e-05, 3.466563066467643e-05, 3.3279986382694915e-05, 3.2412888685939834e-05, 3.136719897156581e-05, 3.093036502832547e-05, 2.9625922252307646e-05, 2.9257877031341195e-05, 2.9086982976878062e-05, 2.8806922273361124e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 0.00017341253987979144, "first_to_last_ratio": 1.4955148931299893, "imbalance_abs_log_ratio": 0.4024705576674927, "last_quartile_mean": 0.00011595507385209203, "mean": 0.00013990553861731314, "normalized": [ 1.3768818258178377, 1.3340879911950096, 1.2910054180628048, 1.2527236427111486, 1.2147542166907819, 1.1769889670731195, 1.1474411155803916, 1.1220953663134672, 1.0944075691302455, 1.0773007485615413, 1.0508678613623372, 1.0320596054419082, 1.0208161736522385, 1.000559648637946, 0.9805197815181176, 0.9663128275349431, 0.948744191771126, 0.9319305822463463, 0.9182019814348666, 0.9075038868226692, 0.8943650364968407, 0.886262676120959, 0.8757907045344768, 0.8679002174812178, 0.856288373659771, 0.8478178092769747, 0.8387215581135022, 0.8313975747674917, 0.8224084044020752, 0.8161864847722404, 0.8112200992010112, 0.8064376596145924 ], "peak_layer": 1, "peak_normalized": 1.3768818258178377, "population_cv": 0.16325888721442564, "rest_mean": 0.0001431150389921472, "spike_contrast": 0.8564734877369171, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001225742365932092, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0001926333934534341, 0.00018664629897102714, 0.00018061880837194622, 0.0001752629759721458, 0.00016995084297377616, 0.0001646672753849998, 0.00016053336730692536, 0.00015698735660407692, 0.00015311368042603135, 0.00015072034148033708, 0.00014702223415952176, 0.00014439085498452187, 0.0001428178366040811, 0.00013998383656144142, 0.00013718014815822244, 0.00013519251660909504, 0.00013273456715978682, 0.00013038225006312132, 0.00012846154277212918, 0.00012696482008323073, 0.00012512662215158343, 0.0001239930570591241, 0.00012252797023393214, 0.000121424047392793, 0.00011979948612861335, 0.00011861440725624561, 0.00011734179133782163, 0.0001163171255029738, 0.00011505949078127742, 0.00011418900976423174, 0.00011349418491590768, 0.00011282509512966499 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 0.00014675183047074825, "first_to_last_ratio": 1.2676545313577883, "imbalance_abs_log_ratio": 0.23716836728277432, "last_quartile_mean": 0.00011576642282307148, "mean": 0.00013041823922321782, "normalized": [ 1.1700698937843759, 1.1560440994564782, 1.1417398043557874, 1.130463200949706, 1.118403537277814, 1.1065304334184665, 1.0939881912841023, 1.0846815122144253, 1.0709468270677047, 1.0579907390333845, 1.0442838370166418, 1.035550673160169, 1.025959021745254, 1.0140203095307772, 1.0062047700154308, 0.9976631870190885, 0.9861765922450596, 0.9770016877798172, 0.9687702939454803, 0.9602422119882037, 0.9504995944091904, 0.9425883202531964, 0.9330014667281973, 0.9259385264566372, 0.9147443798098084, 0.906202796813466, 0.8958065504175365, 0.8885655303249116, 0.8806194435723184, 0.8757905905296257, 0.8713783728577705, 0.8681336045391749 ], "peak_layer": 1, "peak_normalized": 1.1700698937843759, "population_cv": 0.09121429133712369, "rest_mean": 0.0001320278307604086, "spike_contrast": 0.921975648780332, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001217264449223876, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0001525984553154558, 0.0001507692359155044, 0.000148903694935143, 0.00014743302017450333, 0.00014586022007279098, 0.00014431175077334046, 0.00014267601363826543, 0.00014146225294098258, 0.000139670999487862, 0.00013798128929920495, 0.00013619365927297622, 0.00013505469541996717, 0.00013380376913119107, 0.00013224674330558628, 0.00013122745440341532, 0.00013011347618885338, 0.00012861541472375393, 0.00012741883983835578, 0.0001263453159481287, 0.0001252330985153094, 0.0001239624834852293, 0.00012293070903979242, 0.00012168040848337114, 0.00012075927224941552, 0.00011929935135412961, 0.00011818537313956767, 0.00011682951299007982, 0.00011588515189941972, 0.00011484883725643158, 0.00011421906674513593, 0.00011364363308530301, 0.00011322045611450449 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00030004180916876066, "first_to_last_ratio": 2.2837506327398005, "imbalance_abs_log_ratio": 0.8258191057718037, "last_quartile_mean": 0.00013138116082700435, "mean": 0.00019473525253488333, "normalized": [ 1.9000096511818072, 1.783700741635724, 1.6708066910504589, 1.570366884717585, 1.4783058799917985, 1.376185321596543, 1.3064973297630273, 1.2402700043336092, 1.1767953849206394, 1.1505341943958838, 1.0771542571420398, 1.026098162637436, 1.013898063759585, 0.9669473025647412, 0.9095398902231254, 0.8755205777387188, 0.842301214161826, 0.8073124727024382, 0.7791991818813049, 0.760574607765185, 0.7402055397447304, 0.7308143413325056, 0.7152667122155245, 0.7043714155678553, 0.6942467749765324, 0.6868749151283189, 0.680655564574882, 0.676363713518914, 0.6701061029149729, 0.6672276319278244, 0.6636177363814285, 0.6582317375530352 ], "peak_layer": 1, "peak_normalized": 1.9000096511818072, "population_cv": 0.35930746026536914, "rest_mean": 0.00020494150898971216, "spike_contrast": 0.6812747128050888, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013962146767880766, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0003699988592416048, 0.0003473494143690914, 0.0003253649629186839, 0.0003058057918678969, 0.0002878782688640058, 0.0002679917961359024, 0.0002544210874475539, 0.00024152429250534624, 0.00022916354646440595, 0.000224049566895701, 0.00020975990628357977, 0.00019981748482678086, 0.000197441695490852, 0.0001882987271528691, 0.00017711948021315038, 0.00017049472080543637, 0.00016402573965024203, 0.00015721219824627042, 0.00015173754945863038, 0.00014811068831477314, 0.00014414411270990968, 0.0001423153153154999, 0.0001392876438330859, 0.00013716594548895955, 0.00013519432104658335, 0.00013375876005738974, 0.00013254763325676322, 0.0001317118585575372, 0.00013049328117631376, 0.00012993274140171707, 0.00012922976748086512, 0.00012818092363886535 ] } } }, "post_mlp_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "7fd9c911e93175896ad618842c120e77b3f017c17b7eec9c2bfb01fc6dbc8ac2", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.0034885562490671873, 0.004831995815038681, 0.005952076520770788, 0.006925336085259914, 0.00385080324485898, 0.005542940925806761, 0.006729288958013058, 0.007927131839096546, 0.004081308841705322, 0.005811207927763462, 0.007534808944910765, 0.008581922389566898, 0.00499920267611742, 0.007102519739419222, 0.008542952127754688, 0.010447698645293713, 0.005041283555328846, 0.007636935915797949, 0.009387043304741383, 0.011020516976714134, 0.005807018838822842, 0.008367395959794521, 0.01024632342159748, 0.012378672137856483, 0.006453148555010557, 0.009127611294388771, 0.011037957854568958, 0.012611051090061665, 0.007010402157902718, 0.009794782847166061, 0.012445803731679916, 0.014087481424212456 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.00015858902861509705, "first_to_last_ratio": 3.757869257587208, "imbalance_abs_log_ratio": 1.3238521099622325, "last_quartile_mean": 4.220184837322449e-05, "mean": 8.57325238712292e-05, "normalized": [ 2.5462257183789148, 2.3032637538343064, 2.0924289533379263, 1.8886492391629932, 1.7031256747388024, 1.5374176267930122, 1.4224231993460965, 1.3049597038569594, 1.2055618292025094, 1.169245405308266, 1.0769812048488787, 1.0144334047898542, 0.9913512297333369, 0.9317090592952362, 0.8577499774884012, 0.81555804918997, 0.7682725064221458, 0.7217807351250844, 0.6783324192661732, 0.6511800118936032, 0.6192916242842709, 0.6053728292385717, 0.5848720876072855, 0.5718124973482859, 0.5462302773879167, 0.5275173250132701, 0.5112998770983732, 0.49691833866078555, 0.47721658788927634, 0.46551989467389665, 0.46012635628943105, 0.45317260249616614 ], "peak_layer": 1, "peak_normalized": 2.5462257183789148, "population_cv": 0.5686111305901927, "rest_mean": 9.231303704803361e-05, "spike_contrast": 0.5437775023084314, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.019775271648541e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00021829435718245804, 0.00019746461475733668, 0.0001793892151908949, 0.0001619186659809202, 0.00014601326256524771, 0.00013180669338908046, 0.00012194793089292943, 0.00011187748896190897, 0.00010335585830034688, 0.00010024235962191597, 9.233231685357168e-05, 8.69699360919185e-05, 8.499104296788573e-05, 7.987776916706935e-05, 7.353707042057067e-05, 6.991984992055222e-05, 6.586594099644572e-05, 6.188008410390466e-05, 5.8155150327365845e-05, 5.582730591413565e-05, 5.309343396220356e-05, 5.190014053368941e-05, 5.014256021240726e-05, 4.90229285787791e-05, 4.682970029534772e-05, 4.522539165918715e-05, 4.383502891869284e-05, 4.2602063331287354e-05, 4.0912982512963936e-05, 3.991019548266195e-05, 3.944779382436536e-05, 3.885163096128963e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.00021242368711682502, "first_to_last_ratio": 1.8345658968465297, "imbalance_abs_log_ratio": 0.6068078850437921, "last_quartile_mean": 0.0001157896194854402, "mean": 0.0001511171253696375, "normalized": [ 1.7717451736709768, 1.636674793183293, 1.5242755762025908, 1.4186515254985965, 1.3264266232459345, 1.2453738409548603, 1.1893512719369177, 1.1330135273991528, 1.085860457802012, 1.0628542818802846, 1.023252039669589, 0.9942070699734702, 0.9784308644775869, 0.9531083940389613, 0.9232390376562555, 0.9040072643238835, 0.8853775311228679, 0.8680181295617843, 0.8515367513400655, 0.8395795332993111, 0.8267439067367663, 0.8185133286645813, 0.8091730400239778, 0.800791325997512, 0.7909899739933552, 0.7821072339266707, 0.7753403490721149, 0.7682422073314642, 0.7604971517068638, 0.7549863467424895, 0.7508596466770359, 0.7467718018887739 ], "peak_layer": 1, "peak_normalized": 1.7717451736709768, "population_cv": 0.2714186609524326, "rest_mean": 0.00015645541295978345, "spike_contrast": 0.7816308178118705, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012229037238284946, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0002677410375326872, 0.0002473295899108052, 0.00023034414334688336, 0.00021438254043459892, 0.00020044577831868082, 0.00018819731485564262, 0.00017973134526982903, 0.000171217747265473, 0.0001640921109355986, 0.00016061548376455903, 0.00015463090676348656, 0.00015024171443656087, 0.0001478576596127823, 0.00014403100067283958, 0.00013951722939964384, 0.00013661097909789532, 0.00013379570737015456, 0.0001311724045081064, 0.0001286817860091105, 0.00012687484559137374, 0.0001249351626029238, 0.00012369138130452484, 0.00012227990373503417, 0.00012101328320568427, 0.00011953213106608018, 0.0001181897969217971, 0.0001171672047348693, 0.00011609455395955592, 0.00011492414341773838, 0.0001140913664130494, 0.00011346775136189535, 0.00011285000800853595 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.18837944976985455, "first_to_last_ratio": 1.8345658800697515, "imbalance_abs_log_ratio": 0.6068078758989703, "last_quartile_mean": 0.10268339328467846, "mean": 0.1340121782850474, "normalized": [ 1.7717451751532867, 1.6366747548098712, 1.5242756140248972, 1.4186514377480737, 1.3264266357207033, 1.2453739091726768, 1.1893513017871604, 1.1330134634090196, 1.0858604665419038, 1.0628542749024097, 1.0232520332438115, 0.9942070832344971, 0.9784308561813967, 0.9531084116998004, 0.9232390798282929, 0.9040072094634526, 0.8853775581660035, 0.8680181154203164, 0.8515367049614931, 0.8395794986042229, 0.8267439263823189, 0.8185133396785964, 0.809173050926764, 0.8007913534932437, 0.7909900046670136, 0.782107217826699, 0.7753403703776178, 0.7682422246066585, 0.7604971603471317, 0.7549863443945886, 0.7508596538022533, 0.7467717694238248 ], "peak_layer": 1, "peak_normalized": 1.7717451751532867, "population_cv": 0.27141865758333467, "rest_mean": 0.13874622461972413, "spike_contrast": 0.7816308398662988, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.10844832807779312, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.23743543028831482, 0.21933434903621674, 0.20427149534225464, 0.19011656939983368, 0.17775732278823853, 0.16689527034759521, 0.15938755869865417, 0.15183760225772858, 0.14551852643489838, 0.14243541657924652, 0.13712823390960693, 0.1332358568906784, 0.1311216503381729, 0.12772813439369202, 0.1237252801656723, 0.12114797532558441, 0.1186513751745224, 0.1163249984383583, 0.11411628872156143, 0.11251387745141983, 0.11079375445842743, 0.10969075560569763, 0.10843904316425323, 0.107315793633461, 0.10600229352712631, 0.104811891913414, 0.10390505194664001, 0.10295381397008896, 0.1019158810377121, 0.10117736458778381, 0.10062433779239655, 0.10007651150226593 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 6.873207303215167e-05, "first_to_last_ratio": 2.248757814140914, "imbalance_abs_log_ratio": 0.8103779811585758, "last_quartile_mean": 3.0564462122129044e-05, "mean": 4.6857780716891284e-05, "normalized": [ 1.7400119939228862, 1.6405409364325143, 1.555121247708426, 1.4944054279879166, 1.4327475398601994, 1.3414100714975594, 1.2804101716466747, 1.2499370459040815, 1.192804641985012, 1.1608858830017554, 1.098191983597808, 1.057744066133373, 1.0542717513613309, 1.021869458338179, 0.9775749297615711, 0.9570927445976433, 0.9210091333285576, 0.8815125232807631, 0.847820728512489, 0.8204970180441978, 0.7871838673851995, 0.7759041238100245, 0.7506938949785397, 0.7421070509174572, 0.7072987335050381, 0.6891811912847667, 0.6695407652827761, 0.6616333377893114, 0.6345027924930273, 0.6212629426015038, 0.6200302724102036, 0.614801730639213 ], "peak_layer": 1, "peak_normalized": 1.7400119939228862, "population_cv": 0.32175451551289735, "rest_mean": 4.9004235565748616e-05, "spike_contrast": 0.7196709452950106, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.5266924533061685e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 8.153310045599937e-05, 7.687210745643824e-05, 7.28695304132998e-05, 7.002452184678987e-05, 6.713537004543468e-05, 6.28554989816621e-05, 5.9997179050697014e-05, 5.8569276006892323e-05, 5.589217835222371e-05, 5.439653614303097e-05, 5.1458839152473956e-05, 4.9563539505470544e-05, 4.9400834541302174e-05, 4.788253500009887e-05, 4.58069916930981e-05, 4.4847241952084005e-05, 4.315644400776364e-05, 4.130572051508352e-05, 3.972699778387323e-05, 3.8446669350378215e-05, 3.688568904181011e-05, 3.635714529082179e-05, 3.5175849916413426e-05, 3.477348946034908e-05, 3.3142448955914006e-05, 3.2293501135427505e-05, 3.13731943606399e-05, 3.1002669857116416e-05, 2.9731392714893445e-05, 2.9111002731951885e-05, 2.905324254243169e-05, 2.8808244678657502e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 0.00017190765356644988, "first_to_last_ratio": 1.4894621368061003, "imbalance_abs_log_ratio": 0.3984150727783497, "last_quartile_mean": 0.00011541592721187044, "mean": 0.00013894661265112518, "normalized": [ 1.3739040544627474, 1.3296102750467884, 1.2883342981472405, 1.2499918159027226, 1.2130627732820631, 1.175544413473833, 1.1474083142393565, 1.1199112790671786, 1.092909618037032, 1.0755929927697745, 1.0512861461262244, 1.0312571117882559, 1.0194388253067554, 1.0019061415077835, 0.980344379776871, 0.9647961249933756, 0.9493075664595542, 0.9344817410273759, 0.9198929201794295, 0.9079992279088673, 0.8952920956888009, 0.8869530532606651, 0.8771902552778059, 0.8683889858357474, 0.8579407806493952, 0.8482823463732029, 0.8408658204630546, 0.8330843095877946, 0.8243382853592501, 0.818115422616924, 0.8135611224181157, 0.8090075029660142 ], "peak_layer": 1, "peak_normalized": 1.3739040544627474, "population_cv": 0.16161403917150835, "rest_mean": 0.00014210757001786045, "spike_contrast": 0.8576421569620596, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012187744287075474, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00019089931447524577, 0.00018474484386388212, 0.00017900968668982387, 0.00017368212866131216, 0.0001685509632807225, 0.00016333791427314281, 0.00015942849859129637, 0.00015560787869617343, 0.00015185608936008066, 0.00014945000293664634, 0.00014607264893129468, 0.0001432896824553609, 0.00014164757158141583, 0.0001392114645568654, 0.00013621553080156446, 0.0001340551534667611, 0.00013190307072363794, 0.00012984307250007987, 0.0001278160052606836, 0.00012616341700777411, 0.0001243978040292859, 0.00012323912233114243, 0.00012188261462142691, 0.00012065970804542303, 0.00011920796532649547, 0.00011786595860030502, 0.00011683545744745061, 0.00011575424287002534, 0.00011453901242930442, 0.00011367436673026532, 0.00011304156214464456, 0.00011240885214647278 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 0.00014552358334185556, "first_to_last_ratio": 1.262638843036328, "imbalance_abs_log_ratio": 0.2332038508028458, "last_quartile_mean": 0.00011525352965691127, "mean": 0.00012953498708156985, "normalized": [ 1.1676304712442283, 1.1525836984023852, 1.1391521450612143, 1.1288261090295841, 1.1167600441451297, 1.1054423860149858, 1.094731137320412, 1.082319740349084, 1.067957453000188, 1.0563935463552665, 1.04560972624621, 1.0337959762176818, 1.0235535208863231, 1.0159040893770892, 1.0050628637064216, 0.9950792389307415, 0.9865814183778941, 0.9786230528286792, 0.9704418054120744, 0.9608381133357455, 0.9508407831226554, 0.9429749854457838, 0.9346839821902475, 0.9262276149685831, 0.9170542960948607, 0.9059488475652709, 0.8984648923618161, 0.8908037775288885, 0.8831483920181569, 0.8778579808382188, 0.8744051093280881, 0.8703028022960919 ], "peak_layer": 1, "peak_normalized": 1.1676304712442283, "population_cv": 0.0896642712448568, "rest_mean": 0.00013110964479252557, "spike_contrast": 0.9231344927669946, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012103183544240892, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00015124899800866842, 0.00014929991448298097, 0.00014756005839444697, 0.00014622247545048594, 0.00014465949789155275, 0.00014319346519187093, 0.00014180598373059183, 0.00014019827358424664, 0.00013833785487804562, 0.00013683992438018322, 0.0001354430423816666, 0.00013391274842433631, 0.0001325859921053052, 0.00013159512309357524, 0.00013019080506637692, 0.00012889757636003196, 0.00012779681128449738, 0.00012676592450588942, 0.00012570616672746837, 0.00012446215259842575, 0.00012316714855842292, 0.00012214825255796313, 0.00012107427755836397, 0.00011997888213954866, 0.00011879061639774591, 0.00011735207226593047, 0.0001163826382253319, 0.0001153902558144182, 0.00011439861555118114, 0.00011371332220733166, 0.00011326605454087257, 0.0001127346622524783 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00029742828155576717, "first_to_last_ratio": 2.2781890121807993, "imbalance_abs_log_ratio": 0.8233808344454435, "last_quartile_mean": 0.00013055469935352448, "mean": 0.00019329176166138495, "normalized": [ 1.8891850069388627, 1.7801603926271052, 1.6773544463608039, 1.566500262900107, 1.476456430986117, 1.3754925511314788, 1.3084504582544147, 1.236424592515421, 1.172965600373877, 1.150871569148191, 1.0802294364746856, 1.0319209662753672, 1.0080063247818516, 0.9698985570693867, 0.9067931019504507, 0.8756163462587222, 0.8444435806568995, 0.8113939671049883, 0.7809677247391495, 0.7613442367276676, 0.7402582678385915, 0.7293562882295931, 0.7162426695419845, 0.7062419232587185, 0.6940340549797547, 0.6866100033591334, 0.6831688143117768, 0.6753320518823956, 0.6715919825156708, 0.6686410475337363, 0.6640239865195754, 0.6600233567535221 ], "peak_layer": 1, "peak_normalized": 1.8891850069388627, "population_cv": 0.357915029863326, "rest_mean": 0.00020341356920888993, "spike_contrast": 0.6815376252628051, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001386340009048581, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00036516389809548855, 0.0003440903383307159, 0.0003242187958676368, 0.0003027915954589844, 0.00028538686456158757, 0.00026587137836031616, 0.0002529126941226423, 0.00023899068764876574, 0.00022672458726447076, 0.00022245399304665625, 0.0002087994507746771, 0.00019946182146668434, 0.00019483931828290224, 0.00018747340072877705, 0.00017527563613839447, 0.00016924942610785365, 0.00016322398732881993, 0.00015683576930314302, 0.0001509546273155138, 0.00014716156874783337, 0.0001430858246749267, 0.0001409785618307069, 0.00013844380737282336, 0.00013651074550580233, 0.00013415106514003128, 0.00013271605712361634, 0.00013205090363044292, 0.00013053612201474607, 0.00012981319741811603, 0.00012924280599690974, 0.00012835036613978446, 0.00012757707736454904 ] } } }, "pre_attention_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "de71d4cf2d509a1063f6eac7f50c4885e9a058dee7e4b993ad40f5527f528d00", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.02802751213312149, 0.01413465291261673, 0.014225807972252369, 0.01433455292135477, 0.01440478302538395, 0.009703479707241058, 0.00977881159633398, 0.009858808480203152, 0.009950309991836548, 0.00752287870272994, 0.007571155205368996, 0.007693085819482803, 0.007788818795233965, 0.006353048607707024, 0.006389216054230928, 0.0064707002602517605, 0.00656962301582098, 0.005553483963012695, 0.0056249531917274, 0.005688921548426151, 0.005768724251538515, 0.004966245498508215, 0.005057469941675663, 0.005098348017781973, 0.005216081161051989, 0.0046758754178881645, 0.004727107938379049, 0.004777040798217058, 0.004837613552808762, 0.0044186655431985855, 0.004510571248829365, 0.0045849340967834 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.0001580166108396952, "first_to_last_ratio": 1.9713740353643698, "imbalance_abs_log_ratio": 0.6787307795042399, "last_quartile_mean": 8.015557068574708e-05, "mean": 0.00011843875881822896, "normalized": [ 0.8918679597819287, 1.5814819593588525, 1.5722239987443252, 1.3246067564285042, 1.1252650527356378, 1.512743589128775, 1.3788112428745087, 1.2863035124436846, 1.097095055843533, 1.468154353173571, 1.1523254606068964, 1.0839888571909224, 1.054518217028451, 1.146335080549274, 1.1665208527784754, 0.8931136212023528, 0.8203804970547371, 1.073741223277434, 0.919654742394771, 0.8223971330190447, 0.7711288002244273, 0.8470775332584999, 0.8298711629738247, 0.7662487463744663, 0.7210382462310316, 0.7071376059209309, 0.7189559392079661, 0.727980518763353, 0.6111916361855597, 0.6986829946379066, 0.6244678178312321, 0.6046898327751223 ], "peak_layer": 2, "peak_normalized": 1.5814819593588525, "population_cv": 0.2904715201464359, "rest_mean": 0.00012310891098523926, "spike_contrast": 0.7572151875143335, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.321993711637333e-05, "top_five_layers": [ 2, 3, 6, 10, 7 ], "top_five_spike_overlap": 0 }, "values": [ 0.00010563173418631777, 0.0001873087603598833, 0.00018621225899551064, 0.00015688478015363216, 0.0001332749961875379, 0.00017916747310664505, 0.00016330469225067645, 0.00015234819147735834, 0.00012993857671972364, 0.00017388637934345752, 0.0001364799973089248, 0.0001283862948184833, 0.00012489582877606153, 0.00013577050413005054, 0.00013816128193866462, 0.00010577926877886057, 9.71648478298448e-05, 0.00012717257777694613, 0.00010892276623053476, 9.74036956904456e-05, 9.133153798757121e-05, 0.0001003268116619438, 9.828891052166e-05, 9.07535504666157e-05, 8.539887494407594e-05, 8.375250035896897e-05, 8.515224908478558e-05, 8.622110908618197e-05, 7.238877878990024e-05, 8.275114669231698e-05, 7.396119326585904e-05, 7.161871326388791e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.0001664285200604354, "first_to_last_ratio": 1.9719337364651075, "imbalance_abs_log_ratio": 0.6790146534176558, "last_quartile_mean": 8.439863722742302e-05, "mean": 0.0001245867656507471, "normalized": [ 0.8984334461398115, 1.5927952483854981, 1.5658595485525297, 1.3198735865176374, 1.131213894321221, 1.5125315092450449, 1.3796810696180641, 1.286366052942558, 1.0960606282761096, 1.4536884557404346, 1.1543342746982923, 1.0797559630746076, 1.0497530570909623, 1.1457182987306291, 1.1551962694199982, 0.9023721663305979, 0.8238710779144349, 1.0632503993742801, 0.9211624832492573, 0.8274335806257342, 0.7739032412019898, 0.850990856837025, 0.8281317613098801, 0.7681943949939335, 0.7208251570035091, 0.7089546837555952, 0.7203162530736348, 0.7236174125342815, 0.6145740290254166, 0.6994991973458914, 0.6274381906330746, 0.6042038120380651 ], "peak_layer": 2, "peak_normalized": 1.5927952483854981, "population_cv": 0.2889665774838781, "rest_mean": 0.00012946851159169995, "spike_contrast": 0.7586812913967177, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.822533756960183e-05, "top_five_layers": [ 2, 3, 6, 10, 7 ], "top_five_spike_overlap": 0 }, "values": [ 0.00011193291720701382, 0.0001984412083402276, 0.0001950853766174987, 0.000164438781212084, 0.00014093428035266697, 0.00018844140868168324, 0.00017189000209327787, 0.00016026418597903103, 0.00013655464863404632, 0.00018111034296453, 0.00014381477376446128, 0.0001345233031315729, 0.00013078533811494708, 0.00014274133718572557, 0.00014392216689884663, 0.00011242362961638719, 0.00010264343291055411, 0.0001324669283349067, 0.00011476465442683548, 0.00010308727360097691, 9.641810174798593e-05, 0.00010602219845168293, 0.00010317425767425448, 9.570685506332666e-05, 8.980527491075918e-05, 8.832637104205787e-05, 8.974187221610919e-05, 9.015315299620852e-05, 7.656779052922502e-05, 8.714834257261828e-05, 7.817049481673166e-05, 7.527579873567447e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.14759047608822584, "first_to_last_ratio": 1.9719337947467823, "imbalance_abs_log_ratio": 0.6790146829732503, "last_quartile_mean": 0.07484555337578058, "mean": 0.1104847895912826, "normalized": [ 0.8984334974418968, 1.5927952138130326, 1.565859646322909, 1.3198735575362357, 1.1312139236939245, 1.5125315007911495, 1.3796810264894532, 1.286366010451999, 1.0960606168617568, 1.4536884922319049, 1.1543342013148132, 1.0797558953237962, 1.0497530983161607, 1.1457182562234747, 1.1551962948455465, 0.9023721940039697, 0.8238711061937815, 1.0632504861804335, 0.9211625151546727, 0.8274335809310764, 0.7739032642161865, 0.8509909070635955, 0.8281317391228538, 0.7681943896830757, 0.7208251689669645, 0.7089546589522648, 0.7203162343708721, 0.7236173971219663, 0.6145740294939214, 0.6994991437375198, 0.6274381319743458, 0.6042038211744466 ], "peak_layer": 2, "peak_normalized": 1.5927952138130326, "population_cv": 0.28896658018674404, "rest_mean": 0.11481397047086998, "spike_contrast": 0.7586813040631772, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.08710721284151077, "top_five_layers": [ 2, 3, 6, 10, 7 ], "top_five_spike_overlap": 0 }, "values": [ 0.09926323592662811, 0.1759796440601349, 0.1730036735534668, 0.14582595229148865, 0.12498193234205246, 0.16711172461509705, 0.15243376791477203, 0.14212387800216675, 0.12109802663326263, 0.16061046719551086, 0.1275363713502884, 0.1192966029047966, 0.11598175019025803, 0.12658444046974182, 0.12763161957263947, 0.09969840198755264, 0.0910252258181572, 0.11747300624847412, 0.10177444666624069, 0.09141882508993149, 0.08550453931093216, 0.09402155131101608, 0.09149596095085144, 0.08487379550933838, 0.0796402171254158, 0.0783287063241005, 0.07958398759365082, 0.07994871586561203, 0.06790108233690262, 0.07728401571512222, 0.069322369992733, 0.06675533205270767 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 5.093040158499207e-05, "first_to_last_ratio": 0.8860093039525244, "imbalance_abs_log_ratio": 0.12102782735710801, "last_quartile_mean": 5.748291960117058e-05, "mean": 5.7093861016710434e-05, "normalized": [ 0.4337136137178416, 0.9185569667519601, 0.9444095926066538, 0.8728787927927439, 0.7849140301431445, 1.1296292591692207, 1.0726593944235638, 0.9796135302182263, 0.9386642177229508, 1.3046353648934683, 1.0461832118506482, 1.0032917361864164, 0.9662905330123907, 1.0760080966276753, 1.1828739682986207, 0.9087955578115051, 0.9194941496879511, 1.198292373503509, 1.0946132296674007, 1.0673695485587213, 0.9244260207272295, 1.0525679458300632, 1.125653431968099, 0.9999504891362015, 0.9629027708994021, 1.0063234987710272, 0.9975933861195996, 1.0791525148711678, 0.9412468232151083, 1.085173475571988, 1.0031556955576197, 0.9789667796878816 ], "peak_layer": 10, "peak_normalized": 1.3046353648934683, "population_cv": 0.1441687540611157, "rest_mean": 5.695535414647397e-05, "spike_contrast": 1.0155638391297448, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.784179811598733e-05, "top_five_layers": [ 10, 18, 15, 6, 23 ], "top_five_spike_overlap": 1 }, "values": [ 2.4762384782661684e-05, 5.2443963795667514e-05, 5.3919990023132414e-05, 4.9836020480142906e-05, 4.481377254705876e-05, 6.449489592341706e-05, 6.124226638348773e-05, 5.592991874436848e-05, 5.3591964388033375e-05, 7.448667020071298e-05, 5.973063889541663e-05, 5.728179894504137e-05, 5.516925739357248e-05, 6.143345672171563e-05, 6.75348419463262e-05, 5.1886647270293906e-05, 5.249747118796222e-05, 6.84151382301934e-05, 6.249569560168311e-05, 6.094024865888059e-05, 5.277905074763112e-05, 6.009516800986603e-05, 6.426790059776977e-05, 5.7091034250333905e-05, 5.4975836974335834e-05, 5.74548939766828e-05, 5.695645813830197e-05, 6.161298369988799e-05, 5.373941530706361e-05, 6.19567435933277e-05, 5.727403186028823e-05, 5.58929932594765e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 5.801741053801379e-05, "first_to_last_ratio": 0.954195857595535, "imbalance_abs_log_ratio": 0.046886327142486896, "last_quartile_mean": 6.08024129178375e-05, "mean": 6.296396202287724e-05, "normalized": [ 0.4796364558374551, 0.9568353419130547, 0.9804638447804884, 0.8949545354119591, 0.8237377234766136, 1.1597846916175047, 1.085649359633547, 0.9904450076993521, 0.9595078978373608, 1.2941748206155441, 1.0680753760655664, 1.0231559828609242, 0.9904019047543832, 1.096490963646882, 1.1850351594596897, 0.9422215946487361, 0.9418631353047864, 1.1705024179981143, 1.0926353878857977, 1.0382275689133567, 0.9320833313178482, 1.0784583323883348, 1.0985080196909318, 0.9917906747078287, 0.9476388143731139, 0.9855066582268529, 0.978396290111904, 1.028407839812513, 0.8982055143685478, 1.0282804954549911, 0.9451124964564701, 0.9138123627295472 ], "peak_layer": 10, "peak_normalized": 1.2941748206155441, "population_cv": 0.13431233496292436, "rest_mean": 6.285090866084729e-05, "spike_contrast": 1.0115120295379672, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.357445017783903e-05, "top_five_layers": [ 10, 15, 18, 6, 23 ], "top_five_spike_overlap": 1 }, "values": [ 3.019981159013696e-05, 6.0246144130360335e-05, 6.173388828756288e-05, 5.634988337988034e-05, 5.186579073779285e-05, 7.302463927771896e-05, 6.835678505012766e-05, 6.236234185053036e-05, 6.041441884008236e-05, 8.148637425620109e-05, 6.725025741616264e-05, 6.442195444833487e-05, 6.235962791834027e-05, 6.903941539349034e-05, 7.461450877599418e-05, 5.9326004702597857e-05, 5.930343468207866e-05, 7.369946979451925e-05, 6.879665306769311e-05, 6.537092122016475e-05, 5.8687659475253895e-05, 6.790400948375463e-05, 6.916641723364592e-05, 6.244707037694752e-05, 5.966709431959316e-05, 6.205140380188823e-05, 6.160370685392991e-05, 6.475263216998428e-05, 5.655457789544016e-05, 6.474461406469345e-05, 5.950802733423188e-05, 5.753724690293893e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 1.7888660295284353e-05, "first_to_last_ratio": 0.41855462748958633, "imbalance_abs_log_ratio": 0.8709478659353102, "last_quartile_mean": 4.2739129185065394e-05, "mean": 3.1986895351110434e-05, "normalized": [ 0.2593955501951102, 0.4997829486650695, 0.48480994473028793, 0.4968760363401999, 0.5474125689219601, 0.7788406177150305, 0.7554473604850604, 0.6514322393717875, 0.7578135841604356, 0.9674810188795085, 0.9032772556875033, 0.9019220651946442, 0.8311356963419073, 0.9587970177975769, 1.1401394869948454, 0.8953886492354074, 1.0262272842074007, 1.1413916918814533, 1.174037958462901, 1.194927715786966, 1.0267140632187253, 1.3434898267078448, 1.3729067467361273, 1.2011929483563188, 1.1735959904166378, 1.3266745689353452, 1.2739952988364465, 1.4049535244931144, 1.2752889026856997, 1.4680979746684197, 1.4182292846928506, 1.3483241791974143 ], "peak_layer": 30, "peak_normalized": 1.4680979746684197, "population_cv": 0.3184955307507873, "rest_mean": 3.066251991657299e-05, "spike_contrast": 1.276428773763581, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.913852269761264e-05, "top_five_layers": [ 30, 31, 28, 23, 32 ], "top_five_spike_overlap": 1 }, "values": [ 8.297258318634704e-06, 1.5986504877218977e-05, 1.5507564967265353e-05, 1.589352177688852e-05, 1.7510028555989265e-05, 2.491269333404489e-05, 2.416441566310823e-05, 2.083729486912489e-05, 2.4240103812189773e-05, 3.094671410508454e-05, 2.889303505071439e-05, 2.8849686714238487e-05, 2.6585450541460887e-05, 3.066893987124786e-05, 3.6469522456172854e-05, 2.8640703021665104e-05, 3.282582474639639e-05, 3.6509576602838933e-05, 3.755382931558415e-05, 3.822202779701911e-05, 3.2841395295690745e-05, 4.2974068492185324e-05, 4.391502443468198e-05, 3.842243313556537e-05, 3.7539692129939795e-05, 4.243620060151443e-05, 4.0751154301688075e-05, 4.494010136113502e-05, 4.0792532672639936e-05, 4.695989628089592e-05, 4.5364751713350415e-05, 4.3128704419359565e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00022091567916504573, "first_to_last_ratio": 1.232986382535498, "imbalance_abs_log_ratio": 0.20943917994887448, "last_quartile_mean": 0.00017917122386279516, "mean": 0.0002198539991695725, "normalized": [ 0.4672441326905094, 1.0299890566632133, 1.08986441571188, 1.0084939820045402, 0.8814237539069011, 1.2693826047962031, 1.1886514905905403, 1.1035827467935222, 1.0473307057033197, 1.4274044616309502, 1.1442998336482333, 1.0911283607583675, 1.1094038041384326, 1.1946873974128753, 1.2511466100570947, 0.9923165311956169, 0.9239703707694655, 1.2473578196113966, 1.1181336030529359, 1.0160745414007064, 0.9397394354798955, 0.9905446518023159, 1.0121913654482901, 0.9359943959598221, 0.8839446941554127, 0.8510229499852975, 0.8648513533654478, 0.8786898836618169, 0.7455559493639876, 0.8399702482950041, 0.7392957942269532, 0.7163130557190521 ], "peak_layer": 10, "peak_normalized": 1.4274044616309502, "population_cv": 0.18952709251819305, "rest_mean": 0.0002217885959436427, "spike_contrast": 0.9441746799407325, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00020940717658959329, "top_five_layers": [ 10, 6, 15, 18, 14 ], "top_five_spike_overlap": 0 }, "values": [ 0.00010272549116052687, 0.00022644721320830286, 0.00023961105034686625, 0.00022172143508214504, 0.0001937845372594893, 0.0002790788421407342, 0.0002613297838252038, 0.00024262708029709756, 0.00023025984410196543, 0.000313820579322055, 0.00025157889467664063, 0.00023988893372006714, 0.00024390686303377151, 0.000262656802078709, 0.00027506958576850593, 0.0002181647578254342, 0.00020313858112785965, 0.00027423660503700376, 0.0002458261442370713, 0.00022338805138133466, 0.00020660547306761146, 0.00021777520305477083, 0.00022253431961871684, 0.00020578211115207523, 0.0001943387760547921, 0.00018710079893935472, 0.0001901410287246108, 0.00019318348495289683, 0.00016391345707233995, 0.00018467081827111542, 0.00016253713693004102, 0.00015748428995721042 ] } } }, "pre_mlp_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "2999a257265024350fd2665e2a8d452a03cd662193b4245d15290408ad29922d", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.014061377383768559, 0.014165290631353855, 0.014281988143920898, 0.014350161887705326, 0.00966146495193243, 0.009741888381540775, 0.009820535778999329, 0.00991273857653141, 0.0074927168898284435, 0.007546746172010899, 0.00766373984515667, 0.007762390188872814, 0.006325984839349985, 0.006375366821885109, 0.006445344537496567, 0.006555322092026472, 0.005531130358576775, 0.005611324217170477, 0.0056777093559503555, 0.00574630219489336, 0.004946626722812653, 0.0050392793491482735, 0.005084683652967215, 0.005203457549214363, 0.004667261149734259, 0.004717131145298481, 0.004768412560224533, 0.004826710093766451, 0.0044052014127373695, 0.004504534415900707, 0.004574235528707504, 0.004644096828997135 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 3.729895206561196e-05, "first_to_last_ratio": 1.2469721805352876, "imbalance_abs_log_ratio": 0.22071835733540474, "last_quartile_mean": 2.9911615229138988e-05, "mean": 3.340667609563752e-05, "normalized": [ 1.308752778077389, 1.2456464596610868, 1.0236956931411572, 0.9922285647386262, 1.26903181062375, 1.1170846373362797, 1.0287316533628064, 0.9469237624012169, 1.1300261751959062, 1.1132698789554223, 1.0126240729566007, 0.9379202574439188, 1.1130503370514597, 1.070713488608195, 0.9897642722064988, 0.922085688720865, 1.036831181462568, 0.9898105546019274, 0.9395130252451869, 0.9001967292772246, 0.9975952535130206, 0.9404117749145336, 0.9224862220158919, 0.8885787590271026, 0.9461670180110204, 0.8974372095121397, 0.8904691498559152, 0.8684437934704528, 0.9244147277585869, 0.9015689206268677, 0.8740901912628669, 0.8604359589635168 ], "peak_layer": 1, "peak_normalized": 1.308752778077389, "population_cv": 0.1164905978879517, "rest_mean": 3.378375206213807e-05, "spike_contrast": 0.9285666618330386, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.137046587653458e-05, "top_five_layers": [ 1, 5, 2, 9, 6 ], "top_five_spike_overlap": 0 }, "values": [ 4.3721080146497115e-05, 4.161290780757554e-05, 3.419827044126578e-05, 3.314705827506259e-05, 4.239413465256803e-05, 3.73180846509058e-05, 3.4366505133220926e-05, 3.1633575417799875e-05, 3.7750418414361775e-05, 3.719064625329338e-05, 3.3828404411906376e-05, 3.1332798243965954e-05, 3.718331208801828e-05, 3.5768978705164045e-05, 3.306473445263691e-05, 3.080381793552078e-05, 3.463708344497718e-05, 3.3066280593629926e-05, 3.138600732199848e-05, 3.007258055731654e-05, 3.332634150865488e-05, 3.1416031561093405e-05, 3.0817198421573266e-05, 2.9684462788281962e-05, 3.1608295103069395e-05, 2.9980394174344838e-05, 2.9747614462394267e-05, 2.9011820515734144e-05, 3.0881623388268054e-05, 3.01184209092753e-05, 2.9200447897892445e-05, 2.874430538213346e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 6.705871965095866e-05, "first_to_last_ratio": 0.7180297289018083, "imbalance_abs_log_ratio": 0.33124430563536084, "last_quartile_mean": 9.33926785364747e-05, "mean": 8.228966771639534e-05, "normalized": [ 0.8685826408392191, 0.8249006353131848, 0.7154905339803773, 0.6884730939760333, 0.9470487095509537, 0.8638069620260335, 0.8256155016985178, 0.7853664463613442, 0.9851686390305121, 0.9692511243547932, 0.9083804947955202, 0.8729855457896263, 1.0422548534804916, 1.0255492616133721, 0.9871528463964067, 0.9398223255303129, 1.0958375595009802, 1.0726325633807163, 1.0495259774381736, 1.0230838788771643, 1.1576441025653552, 1.1142532603054807, 1.1008567438213033, 1.056908774400763, 1.159063578749664, 1.1192434432436902, 1.1032241584740294, 1.1060435702772593, 1.1786235078649367, 1.1628365878275773, 1.141840825041133, 1.1085318534950752 ], "peak_layer": 29, "peak_normalized": 1.1786235078649367, "population_cv": 0.1341844256762087, "rest_mean": 8.049536754023629e-05, "spike_contrast": 1.1426606459269575, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.197888866765424e-05, "top_five_layers": [ 29, 30, 25, 21, 31 ], "top_five_spike_overlap": 2 }, "values": [ 7.14753768988885e-05, 6.788079917896539e-05, 5.887747829547152e-05, 5.66542221349664e-05, 7.793232362018898e-05, 7.108238787623122e-05, 6.793962529627606e-05, 6.462754390668124e-05, 8.106919995043427e-05, 7.975935295689851e-05, 7.475032907677814e-05, 7.183769048424438e-05, 8.576680556870997e-05, 8.4392107964959e-05, 8.123247971525416e-05, 7.733766688033938e-05, 9.017610864248127e-05, 8.826657722238451e-05, 8.636514394311234e-05, 8.418923243880272e-05, 9.526214853394777e-05, 9.169153054244816e-05, 9.0589135652408e-05, 8.697267185198143e-05, 9.537895675748587e-05, 9.210217103827745e-05, 9.078394941752777e-05, 9.101595787797123e-05, 9.698853682493791e-05, 9.56894364207983e-05, 9.396170207764953e-05, 9.122071787714958e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.05946834245696664, "first_to_last_ratio": 0.7180297222365345, "imbalance_abs_log_ratio": 0.33124431491808765, "last_quartile_mean": 0.08282156102359295, "mean": 0.07297530013602227, "normalized": [ 0.8685826122359729, 0.8249005917343738, 0.71549052911611, 0.6884731350635278, 0.9470486699630107, 0.8638070113577097, 0.8256154769796886, 0.7853664269052657, 0.9851686341386658, 0.9692511556795396, 0.9083805454214465, 0.8729855579949275, 1.0422548040385637, 1.0255492365750694, 0.9871528950273681, 0.9398223256645052, 1.0958375053304956, 1.07263252629438, 1.0495259690486791, 1.0230837911446407, 1.1576441435704576, 1.1142533045483842, 1.1008568124470575, 1.0569088284972943, 1.1590636022344982, 1.119243411839159, 1.1032241423789504, 1.1060435591240758, 1.1786235037173105, 1.1628366076958818, 1.141840809884862, 1.1085318743481278 ], "peak_layer": 29, "peak_normalized": 1.1786235037173105, "population_cv": 0.13418442668798974, "rest_mean": 0.07138409617322462, "spike_contrast": 1.1426607032635474, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.08156780153512955, "top_five_layers": [ 29, 30, 25, 21, 31 ], "top_five_spike_overlap": 2 }, "values": [ 0.06338507682085037, 0.0601973682641983, 0.05221313610672951, 0.050241533666849136, 0.0691111609339714, 0.06303657591342926, 0.060249537229537964, 0.05731235072016716, 0.07189297676086426, 0.07073139399290085, 0.06628934293985367, 0.06370638310909271, 0.07605885714292526, 0.0748397633433342, 0.07203777879476547, 0.06858381628990173, 0.07996907085180283, 0.07827568054199219, 0.07658947259187698, 0.07465984672307968, 0.0844794288277626, 0.08131296932697296, 0.0803353562951088, 0.07712823897600174, 0.08458301424980164, 0.08167712390422821, 0.08050811290740967, 0.08071386069059372, 0.0860104039311409, 0.08485835045576096, 0.08332617580890656, 0.08089544624090195 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 7.408341275549901e-06, "first_to_last_ratio": 0.49124437919492936, "imbalance_abs_log_ratio": 0.7108135577065074, "last_quartile_mean": 1.5080765479069669e-05, "mean": 1.1844782648040564e-05, "normalized": [ 0.5657990507718705, 0.5382233871697658, 0.49942145823009426, 0.48906984415931193, 0.8062921756393574, 0.7122953827323945, 0.7013508394120195, 0.6911626993599347, 0.8940675881372303, 0.868722127857376, 0.9735911331661544, 0.854855860776803, 0.989483587379368, 1.0059088486455374, 1.0303729006301217, 0.9941025538400318, 1.1620912657632405, 1.127012997009919, 1.113401270263767, 1.066097770938846, 1.1826034542812638, 1.1643547935214302, 1.2049778184223279, 1.1791491538213539, 1.2365172470752577, 1.219921447624247, 1.2337761202582145, 1.2288278254433027, 1.3039217543779833, 1.3631114810840155, 1.2856635723507386, 1.3138525898567213 ], "peak_layer": 30, "peak_normalized": 1.3631114810840155, "population_cv": 0.2549101540724733, "rest_mean": 1.1420299651271543e-05, "spike_contrast": 1.237882653019464, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.4136990830593277e-05, "top_five_layers": [ 30, 32, 29, 31, 25 ], "top_five_spike_overlap": 1 }, "values": [ 6.701766778860474e-06, 6.3751390371180605e-06, 5.915538622502936e-06, 5.7929260037781205e-06, 9.550355571263935e-06, 8.436983989668079e-06, 8.307348252856173e-06, 8.186671948351432e-06, 1.0590036254143342e-05, 1.0289824786013924e-05, 1.1531975360412616e-05, 1.0125581866304856e-05, 1.1720218026312068e-05, 1.1914771675947122e-05, 1.220454305439489e-05, 1.1774928680097219e-05, 1.3764718460151926e-05, 1.334922399109928e-05, 1.3187996046326589e-05, 1.2627696378331166e-05, 1.4007680874783546e-05, 1.379152945446549e-05, 1.4272700354922563e-05, 1.3966765436634887e-05, 1.4646278032159898e-05, 1.4449704394792207e-05, 1.4613809980801307e-05, 1.4555198504240252e-05, 1.5444669770658948e-05, 1.614575921848882e-05, 1.5228405572997872e-05, 1.5562298358418047e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 5.6029601637419546e-05, "first_to_last_ratio": 0.6070213107493427, "imbalance_abs_log_ratio": 0.49919138022039955, "last_quartile_mean": 9.230252817360451e-05, "mean": 7.840498903988191e-05, "normalized": [ 0.6598133090619259, 0.6419677920829977, 0.6005256373772865, 0.5913463573822231, 0.8562376513989928, 0.8062968559250673, 0.7923156566726156, 0.768439227921366, 0.9790122729268586, 0.9665730330981341, 0.9163710065350301, 0.8881686280668529, 1.0631190324760253, 1.0511030519715439, 1.0188080221730096, 0.9712812246823179, 1.1361272291219966, 1.1138561419734825, 1.0918134348286874, 1.0639217496772315, 1.203414696914147, 1.1592426077267617, 1.1442664103435203, 1.097952968622282, 1.2042281788778202, 1.1625445130141736, 1.1452986211445626, 1.1473932954411665, 1.2233942816523278, 1.205318760675343, 1.1829469858778916, 1.146901364356358 ], "peak_layer": 29, "peak_normalized": 1.2233942816523278, "population_cv": 0.1898597249088972, "rest_mean": 7.605543874820728e-05, "spike_contrast": 1.1977126437531995, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.109256061492487e-05, "top_five_layers": [ 29, 30, 25, 21, 31 ], "top_five_spike_overlap": 2 }, "values": [ 5.173265526536852e-05, 5.033347770222463e-05, 4.708420601673424e-05, 4.636450466932729e-05, 6.713330367347226e-05, 6.321769615169615e-05, 6.212150037754327e-05, 6.024946924299002e-05, 7.675944652874023e-05, 7.578414806630462e-05, 7.184805872384459e-05, 6.963685154914856e-05, 8.335383608937263e-05, 8.241172326961532e-05, 7.987963181221858e-05, 7.615329377586022e-05, 8.907804294722155e-05, 8.733187860343605e-05, 8.560362039133906e-05, 8.341677312273532e-05, 9.435371612198651e-05, 9.089040395338088e-05, 8.971619536168873e-05, 8.608499047113582e-05, 9.441749716643244e-05, 9.114928980125114e-05, 8.97971258382313e-05, 8.996135875349864e-05, 9.592021524440497e-05, 9.450300422031432e-05, 9.274894546251744e-05, 8.992278890218586e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 4.8511469685763586e-05, "first_to_last_ratio": 0.5271352332510719, "imbalance_abs_log_ratio": 0.6402981537748061, "last_quartile_mean": 9.202850924339145e-05, "mean": 7.532146378252946e-05, "normalized": [ 0.548104082267993, 0.5416830199005969, 0.528884702689083, 0.5247448675895723, 0.7777813723908718, 0.7487613756360456, 0.7496059866168319, 0.7329069161583202, 0.9480676391902623, 0.9399123881532756, 0.9000936195631334, 0.883441930780847, 1.0560589316963704, 1.0534552095740957, 1.032095781768693, 0.9835620691098637, 1.1601754560133901, 1.1467788545782247, 1.1233340534265408, 1.0967375255670042, 1.2396083970871608, 1.1951467072205837, 1.1828066060767042, 1.1317734013234224, 1.244698956870482, 1.2025862582796845, 1.1881970083126314, 1.1882256015362047, 1.2689460104605979, 1.2589162611011824, 1.2307315535498626, 1.1921774555104672 ], "peak_layer": 29, "peak_normalized": 1.2689460104605979, "population_cv": 0.23486334953575583, "rest_mean": 7.254842300123225e-05, "spike_contrast": 1.244629176846487, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.029588400153444e-05, "top_five_layers": [ 29, 30, 25, 21, 31 ], "top_five_spike_overlap": 2 }, "values": [ 4.1284001781605184e-05, 4.080035796505399e-05, 3.9836369978729635e-05, 3.952455153921619e-05, 5.858363147126511e-05, 5.639780283672735e-05, 5.6461420172126964e-05, 5.520362174138427e-05, 7.140984234865755e-05, 7.079557690303773e-05, 6.77963689668104e-05, 6.654213939327747e-05, 7.954390457598493e-05, 7.934778841445222e-05, 7.773896504659206e-05, 7.408333476632833e-05, 8.738611359149218e-05, 8.637706196168438e-05, 8.461116522084922e-05, 8.26078758109361e-05, 9.336911898571998e-05, 9.002019942272455e-05, 8.909072494134307e-05, 8.524682925781235e-05, 9.375254740007222e-05, 9.058055729838088e-05, 8.949673792812973e-05, 8.949889161158353e-05, 9.557887096889317e-05, 9.482341556577012e-05, 9.270050213672221e-05, 8.979655103757977e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 9.61669284151867e-05, "first_to_last_ratio": 0.8297833624837898, "imbalance_abs_log_ratio": 0.18659062131596024, "last_quartile_mean": 0.00011589401856326731, "mean": 0.00010716335873439675, "normalized": [ 0.9380742209455786, 0.8898820858744699, 0.7837417114644826, 0.7583102664636657, 1.0822070619439073, 0.9520749081713696, 0.9107982467806989, 0.8640017807748708, 1.0669538303069324, 1.0266950649381863, 0.9594029855154438, 0.8901506143407661, 1.0795289063552718, 1.0379165674077444, 0.9706971366471856, 0.914979822300863, 1.052939971267624, 1.0225707935964177, 0.9851238030379684, 0.9636652893153618, 1.0990044510799908, 1.0523904893769196, 1.0422234788441622, 1.0049019599938585, 1.0962247910521359, 1.0599378706264646, 1.05039930496957, 1.0583562345651827, 1.1199788667078585, 1.1100529546792268, 1.0850973923636869, 1.0717171382921347 ], "peak_layer": 29, "peak_normalized": 1.1199788667078585, "population_cv": 0.09156717959150794, "rest_mean": 0.00010599351123716958, "spike_contrast": 1.0706366257223145, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00011348053521942347, "top_five_layers": [ 29, 30, 21, 25, 31 ], "top_five_spike_overlap": 2 }, "values": [ 0.00010052718425868079, 9.536275319987908e-05, 8.398839418077841e-05, 8.12630751170218e-05, 0.00011597294360399246, 0.00010202754492638633, 9.760419925441965e-05, 9.25893327803351e-05, 0.00011433835607022047, 0.00011002409155480564, 0.00010281284630764276, 9.539152961224318e-05, 0.000115685943455901, 0.0001112266254494898, 0.0001040231654769741, 9.805231093196198e-05, 0.00011283658386673778, 0.00010958212078548968, 0.00010556917550275102, 0.00010326960909878835, 0.00011777300824178383, 0.0001127776995417662, 0.00011168816854478791, 0.00010768866923172027, 0.00011747513053705916, 0.00011358650226611644, 0.00011256431753281504, 0.00011341700883349404, 0.00012002069706795737, 0.00011895700299646705, 0.00011628268111962825, 0.00011484880815260112 ] } } } }, "selector": { "expected_uniform_count": 4, "identities_unique": true, "identity_order_sha256": "97fd9721a263024f6a746638f32e0ad9a988cd4c111c00e9e178c1022e5c9d5e", "passed": true, "uniform_count": 4, "uniform_indices": [ 49, 51, 53, 55 ], "visit_count": 65, "visits": [ { "branch": "attention", "coefficient": "detached_learned", "group": 1, "index": 0, "kind": "depth", "layer": 1, "offset": 1, "sources": 1, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 1, "index": 1, "kind": "depth", "layer": 1, "offset": 1, "sources": 2, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 1, "index": 2, "kind": "depth", "layer": 2, "offset": 2, "sources": 2, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 1, "index": 3, "kind": "depth", "layer": 2, "offset": 2, "sources": 2, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 1, "index": 4, "kind": "depth", "layer": 3, "offset": 3, "sources": 2, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 1, "index": 5, "kind": "depth", "layer": 3, "offset": 3, "sources": 2, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 1, "index": 6, "kind": "depth", "layer": 4, "offset": 4, "sources": 2, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 1, "index": 7, "kind": "depth", "layer": 4, "offset": 4, "sources": 2, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 2, "index": 8, "kind": "depth", "layer": 5, "offset": 1, "sources": 2, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 2, "index": 9, "kind": "depth", "layer": 5, "offset": 1, "sources": 3, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 2, "index": 10, "kind": "depth", "layer": 6, "offset": 2, "sources": 3, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 2, "index": 11, "kind": "depth", "layer": 6, "offset": 2, "sources": 3, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 2, "index": 12, "kind": "depth", "layer": 7, "offset": 3, "sources": 3, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 2, "index": 13, "kind": "depth", "layer": 7, "offset": 3, "sources": 3, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 2, "index": 14, "kind": "depth", "layer": 8, "offset": 4, "sources": 3, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 2, "index": 15, "kind": "depth", "layer": 8, "offset": 4, "sources": 3, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 3, "index": 16, "kind": "depth", "layer": 9, "offset": 1, "sources": 3, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 3, "index": 17, "kind": "depth", "layer": 9, "offset": 1, "sources": 4, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 3, "index": 18, "kind": "depth", "layer": 10, "offset": 2, "sources": 4, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 3, "index": 19, "kind": "depth", "layer": 10, "offset": 2, "sources": 4, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 3, "index": 20, "kind": "depth", "layer": 11, "offset": 3, "sources": 4, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 3, "index": 21, "kind": "depth", "layer": 11, "offset": 3, "sources": 4, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 3, "index": 22, "kind": "depth", "layer": 12, "offset": 4, "sources": 4, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 3, "index": 23, "kind": "depth", "layer": 12, "offset": 4, "sources": 4, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 4, "index": 24, "kind": "depth", "layer": 13, "offset": 1, "sources": 4, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 4, "index": 25, "kind": "depth", "layer": 13, "offset": 1, "sources": 5, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 4, "index": 26, "kind": "depth", "layer": 14, "offset": 2, "sources": 5, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 4, "index": 27, "kind": "depth", "layer": 14, "offset": 2, "sources": 5, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 4, "index": 28, "kind": "depth", "layer": 15, "offset": 3, "sources": 5, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 4, "index": 29, "kind": "depth", "layer": 15, "offset": 3, "sources": 5, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 4, "index": 30, "kind": "depth", "layer": 16, "offset": 4, "sources": 5, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 4, "index": 31, "kind": "depth", "layer": 16, "offset": 4, "sources": 5, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 5, "index": 32, "kind": "depth", "layer": 17, "offset": 1, "sources": 5, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 5, "index": 33, "kind": "depth", "layer": 17, "offset": 1, "sources": 6, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 5, "index": 34, "kind": "depth", "layer": 18, "offset": 2, "sources": 6, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 5, "index": 35, "kind": "depth", "layer": 18, "offset": 2, "sources": 6, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 5, "index": 36, "kind": "depth", "layer": 19, "offset": 3, "sources": 6, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 5, "index": 37, "kind": "depth", "layer": 19, "offset": 3, "sources": 6, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 5, "index": 38, "kind": "depth", "layer": 20, "offset": 4, "sources": 6, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 5, "index": 39, "kind": "depth", "layer": 20, "offset": 4, "sources": 6, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 6, "index": 40, "kind": "depth", "layer": 21, "offset": 1, "sources": 6, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 6, "index": 41, "kind": "depth", "layer": 21, "offset": 1, "sources": 7, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 6, "index": 42, "kind": "depth", "layer": 22, "offset": 2, "sources": 7, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 6, "index": 43, "kind": "depth", "layer": 22, "offset": 2, "sources": 7, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 6, "index": 44, "kind": "depth", "layer": 23, "offset": 3, "sources": 7, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 6, "index": 45, "kind": "depth", "layer": 23, "offset": 3, "sources": 7, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 6, "index": 46, "kind": "depth", "layer": 24, "offset": 4, "sources": 7, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 6, "index": 47, "kind": "depth", "layer": 24, "offset": 4, "sources": 7, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 7, "index": 48, "kind": "depth", "layer": 25, "offset": 1, "sources": 7, "use_uniform": false }, { "branch": "mlp", "coefficient": "uniform", "group": 7, "index": 49, "kind": "depth", "layer": 25, "offset": 1, "sources": 8, "use_uniform": true }, { "branch": "attention", "coefficient": "detached_learned", "group": 7, "index": 50, "kind": "depth", "layer": 26, "offset": 2, "sources": 8, "use_uniform": false }, { "branch": "mlp", "coefficient": "uniform", "group": 7, "index": 51, "kind": "depth", "layer": 26, "offset": 2, "sources": 8, "use_uniform": true }, { "branch": "attention", "coefficient": "detached_learned", "group": 7, "index": 52, "kind": "depth", "layer": 27, "offset": 3, "sources": 8, "use_uniform": false }, { "branch": "mlp", "coefficient": "uniform", "group": 7, "index": 53, "kind": "depth", "layer": 27, "offset": 3, "sources": 8, "use_uniform": true }, { "branch": "attention", "coefficient": "detached_learned", "group": 7, "index": 54, "kind": "depth", "layer": 28, "offset": 4, "sources": 8, "use_uniform": false }, { "branch": "mlp", "coefficient": "uniform", "group": 7, "index": 55, "kind": "depth", "layer": 28, "offset": 4, "sources": 8, "use_uniform": true }, { "branch": "attention", "coefficient": "detached_learned", "group": 8, "index": 56, "kind": "depth", "layer": 29, "offset": 1, "sources": 8, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 8, "index": 57, "kind": "depth", "layer": 29, "offset": 1, "sources": 9, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 8, "index": 58, "kind": "depth", "layer": 30, "offset": 2, "sources": 9, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 8, "index": 59, "kind": "depth", "layer": 30, "offset": 2, "sources": 9, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 8, "index": 60, "kind": "depth", "layer": 31, "offset": 3, "sources": 9, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 8, "index": 61, "kind": "depth", "layer": 31, "offset": 3, "sources": 9, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 8, "index": 62, "kind": "depth", "layer": 32, "offset": 4, "sources": 9, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 8, "index": 63, "kind": "depth", "layer": 32, "offset": 4, "sources": 9, "use_uniform": false }, { "branch": "output", "coefficient": "detached_learned", "group": null, "index": 64, "kind": "output", "layer": null, "offset": null, "sources": 9, "use_uniform": false } ], "visits_sha256": "86ffbfde38ca8bed5f23de529d214f0cf29ea110bc34c6d2b71e42982072b897" } }, "uniform_group_7_only": { "bits_per_byte": 7.959341947457508, "forward": { "activation_sha256": { "attention_branch_output": "676759a2d7c0615ad7e1f246e90927fb829d00c8b6c3745032ae6b27ab7d902a", "mlp_branch_output": "afe29f3ef4d2992353f0bb59fe7c5c2c5050fca671a53184825a8d63cd83a2bc", "post_attention_state": "31853ffb07b24cc59e774bb20656840b4358b3080b42516d380f2dc0f46404c8", "post_mlp_state": "7fd9c911e93175896ad618842c120e77b3f017c17b7eec9c2bfb01fc6dbc8ac2", "pre_attention_input": "de71d4cf2d509a1063f6eac7f50c4885e9a058dee7e4b993ad40f5527f528d00", "pre_mlp_input": "2999a257265024350fd2665e2a8d452a03cd662193b4245d15290408ad29922d" }, "logits_sha256": "873a309a197bfd08fa523edcb440d2697903a34abc02dad1c7f5991509c3c151", "loss_nats": 5.516995429992676, "mixer_summary_sha256": "8450b24bdc777885a3357dc7b8080e7410acbe96943686433da76f379295133a" }, "loss_nats": 5.516995429992676, "loss_scale": 1.0, "mixers": [ { "branch": "attention", "entropy_mean": -0.0, "group": 1, "latest_source_mass": 1.0, "layer": 1, "max_source_mass": 1.0, "mixer_index": 0, "normalized_entropy": 1.0, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 1.0, "median": 1.0, "p05": 1.0, "p95": 1.0 } ], "sources": 1 }, { "branch": "mlp", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 1, "max_source_mass": 0.5, "mixer_index": 1, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 2, "max_source_mass": 0.5, "mixer_index": 2, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 2, "max_source_mass": 0.5, "mixer_index": 3, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 3, "max_source_mass": 0.5, "mixer_index": 4, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 3, "max_source_mass": 0.5, "mixer_index": 5, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 4, "max_source_mass": 0.5, "mixer_index": 6, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 4, "max_source_mass": 0.5, "mixer_index": 7, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6931471824645996, "group": 2, "latest_source_mass": 0.5, "layer": 5, "max_source_mass": 0.5, "mixer_index": 8, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "completed_group_1", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 5, "max_source_mass": 0.3333333432674408, "mixer_index": 9, "normalized_entropy": 1.0000000180537645, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 6, "max_source_mass": 0.3333333432674408, "mixer_index": 10, "normalized_entropy": 1.0000000180537645, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 6, "max_source_mass": 0.3333333432674408, "mixer_index": 11, "normalized_entropy": 1.0000000180537645, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 7, "max_source_mass": 0.3333333432674408, "mixer_index": 12, "normalized_entropy": 1.0000000180537645, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 7, "max_source_mass": 0.3333333432674408, "mixer_index": 13, "normalized_entropy": 1.0000000180537645, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 8, "max_source_mass": 0.3333333432674408, "mixer_index": 14, "normalized_entropy": 1.0000000180537645, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 8, "max_source_mass": 0.3333333432674408, "mixer_index": 15, "normalized_entropy": 1.0000000180537645, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0986123085021973, "group": 3, "latest_source_mass": 0.3333333432674408, "layer": 9, "max_source_mass": 0.3333333432674408, "mixer_index": 16, "normalized_entropy": 1.0000000180537645, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_2", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 9, "max_source_mass": 0.25, "mixer_index": 17, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 10, "max_source_mass": 0.25, "mixer_index": 18, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 10, "max_source_mass": 0.25, "mixer_index": 19, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 11, "max_source_mass": 0.25, "mixer_index": 20, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 11, "max_source_mass": 0.25, "mixer_index": 21, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 12, "max_source_mass": 0.25, "mixer_index": 22, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 12, "max_source_mass": 0.25, "mixer_index": 23, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3862943649291992, "group": 4, "latest_source_mass": 0.25, "layer": 13, "max_source_mass": 0.25, "mixer_index": 24, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_3", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 13, "max_source_mass": 0.20000000298023224, "mixer_index": 25, "normalized_entropy": 1.0000000186840277, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 14, "max_source_mass": 0.20000000298023224, "mixer_index": 26, "normalized_entropy": 1.0000000186840277, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 14, "max_source_mass": 0.20000000298023224, "mixer_index": 27, "normalized_entropy": 1.0000000186840277, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 15, "max_source_mass": 0.20000000298023224, "mixer_index": 28, "normalized_entropy": 1.0000000186840277, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 15, "max_source_mass": 0.20000000298023224, "mixer_index": 29, "normalized_entropy": 1.0000000186840277, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 16, "max_source_mass": 0.20000000298023224, "mixer_index": 30, "normalized_entropy": 1.0000000186840277, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 16, "max_source_mass": 0.20000000298023224, "mixer_index": 31, "normalized_entropy": 1.0000000186840277, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.6094379425048828, "group": 5, "latest_source_mass": 0.20000000298023224, "layer": 17, "max_source_mass": 0.20000000298023224, "mixer_index": 32, "normalized_entropy": 1.0000000186840277, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_4", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 17, "max_source_mass": 0.1666666716337204, "mixer_index": 33, "normalized_entropy": 1.0000001451965654, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 18, "max_source_mass": 0.1666666716337204, "mixer_index": 34, "normalized_entropy": 1.0000001451965654, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 18, "max_source_mass": 0.1666666716337204, "mixer_index": 35, "normalized_entropy": 1.0000001451965654, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 19, "max_source_mass": 0.1666666716337204, "mixer_index": 36, "normalized_entropy": 1.0000001451965654, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 19, "max_source_mass": 0.1666666716337204, "mixer_index": 37, "normalized_entropy": 1.0000001451965654, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 20, "max_source_mass": 0.1666666716337204, "mixer_index": 38, "normalized_entropy": 1.0000001451965654, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 20, "max_source_mass": 0.1666666716337204, "mixer_index": 39, "normalized_entropy": 1.0000001451965654, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.791759729385376, "group": 6, "latest_source_mass": 0.1666666716337204, "layer": 21, "max_source_mass": 0.1666666716337204, "mixer_index": 40, "normalized_entropy": 1.0000001451965654, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_5", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 21, "max_source_mass": 0.1428571492433548, "mixer_index": 41, "normalized_entropy": 1.00000003408302, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 22, "max_source_mass": 0.1428571492433548, "mixer_index": 42, "normalized_entropy": 1.00000003408302, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 22, "max_source_mass": 0.1428571492433548, "mixer_index": 43, "normalized_entropy": 1.00000003408302, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 23, "max_source_mass": 0.1428571492433548, "mixer_index": 44, "normalized_entropy": 1.00000003408302, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 23, "max_source_mass": 0.1428571492433548, "mixer_index": 45, "normalized_entropy": 1.00000003408302, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 24, "max_source_mass": 0.1428571492433548, "mixer_index": 46, "normalized_entropy": 1.00000003408302, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 24, "max_source_mass": 0.1428571492433548, "mixer_index": 47, "normalized_entropy": 1.00000003408302, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9459102153778076, "group": 7, "latest_source_mass": 0.1428571492433548, "layer": 25, "max_source_mass": 0.1428571492433548, "mixer_index": 48, "normalized_entropy": 1.00000003408302, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_6", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 25, "max_source_mass": 0.125, "mixer_index": 49, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 26, "max_source_mass": 0.125, "mixer_index": 50, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 26, "max_source_mass": 0.125, "mixer_index": 51, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 27, "max_source_mass": 0.125, "mixer_index": 52, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 27, "max_source_mass": 0.125, "mixer_index": 53, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 28, "max_source_mass": 0.125, "mixer_index": 54, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 28, "max_source_mass": 0.125, "mixer_index": 55, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.079441547393799, "group": 8, "latest_source_mass": 0.125, "layer": 29, "max_source_mass": 0.125, "mixer_index": 56, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_7", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 29, "max_source_mass": 0.1111111119389534, "mixer_index": 57, "normalized_entropy": 1.0000000180537645, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 30, "max_source_mass": 0.1111111119389534, "mixer_index": 58, "normalized_entropy": 1.0000000180537645, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 30, "max_source_mass": 0.1111111119389534, "mixer_index": 59, "normalized_entropy": 1.0000000180537645, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 31, "max_source_mass": 0.1111111119389534, "mixer_index": 60, "normalized_entropy": 1.0000000180537645, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 31, "max_source_mass": 0.1111111119389534, "mixer_index": 61, "normalized_entropy": 1.0000000180537645, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 32, "max_source_mass": 0.1111111119389534, "mixer_index": 62, "normalized_entropy": 1.0000000180537645, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 32, "max_source_mass": 0.1111111119389534, "mixer_index": 63, "normalized_entropy": 1.0000000180537645, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "output", "entropy_mean": 2.1972246170043945, "group": null, "latest_source_mass": 0.1111111119389534, "layer": null, "max_source_mass": 0.1111111119389534, "mixer_index": null, "normalized_entropy": 1.0000000180537645, "offset": null, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_8", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 } ], "mode": "uniform_group_7_only", "optimizer_state_after": "b82a349700eda58e2330a22ca2ee1f7fb1406efd47fd9aa306122b368a114192", "optimizer_state_before": "b82a349700eda58e2330a22ca2ee1f7fb1406efd47fd9aa306122b368a114192", "optimizer_state_unchanged": true, "positions": { "attention_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "676759a2d7c0615ad7e1f246e90927fb829d00c8b6c3745032ae6b27ab7d902a", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.0022265110164880753, 0.0021613112185150385, 0.0025601957459002733, 0.0025323182344436646, 0.0027253651060163975, 0.002904455177485943, 0.0029208383057266474, 0.00299624796025455, 0.0030937574338167906, 0.0033804215490818024, 0.0034370317589491606, 0.0038969130255281925, 0.004255065228790045, 0.004119740333408117, 0.003970506135374308, 0.004385598469525576, 0.0041999719105660915, 0.005232699681073427, 0.004664790350943804, 0.004898969549685717, 0.005131533369421959, 0.005069608334451914, 0.0051318020559847355, 0.0058372304774820805, 0.0059690922498703, 0.00554921617731452, 0.0058550527319312096, 0.006255472078919411, 0.006502739619463682, 0.006122888531535864, 0.007071305997669697, 0.006241494789719582 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.00015962477027642308, "first_to_last_ratio": 3.7699040072151497, "imbalance_abs_log_ratio": 1.3270495388586014, "last_quartile_mean": 4.234186599205714e-05, "mean": 8.617327489446325e-05, "normalized": [ 2.5428430303860874, 2.3222064178412967, 2.1011513502014654, 1.8911470095267005, 1.7004447966680725, 1.5353700031893112, 1.4229335846259188, 1.3028620492588128, 1.2049362956920768, 1.1705197150696323, 1.0773612761873523, 1.0133200843661747, 0.9906112963967714, 0.9272530070793641, 0.8550207003106389, 0.8185600560490035, 0.7703816559177782, 0.7173918665339755, 0.6760157258502337, 0.6514535263006792, 0.6184138115448249, 0.6053606047045522, 0.5838821473278404, 0.5697017460729616, 0.5457761369570343, 0.5276086337508958, 0.5104655265174319, 0.4951752820428687, 0.47527333481689216, 0.465789239220051, 0.45896414036514044, 0.45180594922816086 ], "peak_layer": 1, "peak_normalized": 2.5428430303860874, "population_cv": 0.5705341151299013, "rest_mean": 9.280180475562466e-05, "spike_contrast": 0.5428688997682254, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.0379213644191625e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00021912511147093028, 0.00020011213200632483, 0.00018106309289578348, 0.00016296633111778647, 0.00014653289690613747, 0.00013230786134954542, 0.0001226188469445333, 0.00011227188952034339, 0.0001038333066389896, 0.00010086751717608422, 9.283974941354245e-05, 8.732111018616706e-05, 8.536421955795959e-05, 7.990442827576771e-05, 7.367993384832516e-05, 7.053800072753802e-05, 6.63863102090545e-05, 6.182000652188435e-05, 5.825448897667229e-05, 5.6137883802875876e-05, 5.329074338078499e-05, 5.216590579948388e-05, 5.031503678765148e-05, 4.909306517220102e-05, 4.703131708083674e-05, 4.5465763832908124e-05, 4.3988486140733585e-05, 4.2670875700423494e-05, 4.095585973118432e-05, 4.013858415419236e-05, 3.955044303438626e-05, 3.893359826179221e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.00021442661636683624, "first_to_last_ratio": 1.843144421770586, "imbalance_abs_log_ratio": 0.6114730379455638, "last_quartile_mean": 0.0001163373926829081, "mean": 0.00015221274247778638, "normalized": [ 1.7741446897870732, 1.648461275666186, 1.5301464224684427, 1.4217121808823485, 1.3265065496562456, 1.2454420544341354, 1.1897998831173078, 1.1336248235930768, 1.0869089599614004, 1.0643003125599004, 1.0227030055526172, 0.9941765648180964, 0.9788958472869692, 0.9511102301475627, 0.9228060651723192, 0.9052418824754537, 0.8846484396889256, 0.865292475366091, 0.8497311645646503, 0.8388691935254817, 0.8256303537004397, 0.8176325372729673, 0.8076852907347736, 0.8000670639426607, 0.7892588692968658, 0.7814415503461742, 0.7731202151361662, 0.7664994567889658, 0.758490072461763, 0.7529625284402992, 0.7485174912671357, 0.7441725498875049 ], "peak_layer": 1, "peak_normalized": 1.7741446897870732, "population_cv": 0.27364872432853743, "rest_mean": 0.00015762320577373935, "spike_contrast": 0.7803181014868814, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012299624067964033, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00027004742878489196, 0.0002509168116375804, 0.0002329077833564952, 0.00021640271006617695, 0.00020191119983792305, 0.00018957215070258826, 0.0001811027032090351, 0.00017255214333999902, 0.00016544139361940324, 0.00016200006939470768, 0.00015566842921543866, 0.0001513263414381072, 0.00014900042151566595, 0.0001447710965294391, 0.00014046284195501357, 0.0001377893495373428, 0.00013465476513374597, 0.00013170854072086513, 0.00012933991092722863, 0.0001276865805266425, 0.00012567146040964872, 0.00012445409083738923, 0.00012293999316170812, 0.00012178040196886286, 0.00012013525702059269, 0.00011894536146428436, 0.00011767874821089208, 0.00011667098442558199, 0.00011545185407157987, 0.00011461049143690616, 0.00011393390013836324, 0.00011327254469506443 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.19015566632151604, "first_to_last_ratio": 1.8431444167877922, "imbalance_abs_log_ratio": 0.6114730352421437, "last_quartile_mean": 0.10316916275769472, "mean": 0.13498378219082952, "normalized": [ 1.7741447141792424, 1.6484611693507762, 1.5301464164964773, 1.421712099711509, 1.3265065258282553, 1.245442086656075, 1.1897999853143832, 1.1336247998774232, 1.0869090145314644, 1.0643003539468234, 1.0227030148964142, 0.9941765578539046, 0.978895844117059, 0.9511102293531465, 0.9228060470726628, 0.9052419251832852, 0.8846484748200404, 0.8652925213936268, 0.8497311905993837, 0.8388692019864461, 0.8256303029992552, 0.8176324960789627, 0.8076852727084025, 0.800067049482784, 0.789258877083647, 0.7814415614650243, 0.7731201949085273, 0.7664994759042474, 0.758490077799444, 0.7529625178867759, 0.7485174642151933, 0.7441725362993381 ], "peak_layer": 1, "peak_normalized": 1.7741447141792424, "population_cv": 0.2736487184871982, "rest_mean": 0.13978183573042904, "spike_contrast": 0.7803180757143806, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.10907429307699204, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.23948076367378235, 0.2225155234336853, 0.20654495060443878, 0.1919080764055252, 0.17905686795711517, 0.16811448335647583, 0.16060370206832886, 0.1530209630727768, 0.14671508967876434, 0.14366328716278076, 0.13804832100868225, 0.13419771194458008, 0.1321350634098053, 0.12838445603847504, 0.12456385046243668, 0.12219297885894775, 0.11941319704055786, 0.116800457239151, 0.11469992995262146, 0.11323373764753342, 0.11144670099020004, 0.11036712676286697, 0.10902441293001175, 0.10799607634544373, 0.10653714835643768, 0.10548193752765656, 0.10435868799686432, 0.1034649983048439, 0.10238385945558548, 0.10163772851228714, 0.10103771835565567, 0.10045122355222702 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 6.883365585963475e-05, "first_to_last_ratio": 2.2497546817252747, "imbalance_abs_log_ratio": 0.8108211799277664, "last_quartile_mean": 3.059607183786284e-05, "mean": 4.691841348858361e-05, "normalized": [ 1.7359303646705315, 1.6475859007944922, 1.5536416135247637, 1.4921074478374423, 1.436565604934822, 1.343834173349448, 1.2787439414688093, 1.2483315192759759, 1.191227587726315, 1.1603086071531024, 1.0995306735006558, 1.059977946939621, 1.0583984121521266, 1.0176934649371145, 0.9754909410255406, 0.9631621800279858, 0.9242301483750994, 0.8752327792179452, 0.8454160845678106, 0.8232909667825862, 0.7863453139125789, 0.77746685672799, 0.7497386589361914, 0.7388508716017261, 0.7093180450413769, 0.6908354487677191, 0.6685471901846501, 0.6592354484095886, 0.631440334976849, 0.6235901145414114, 0.6199506557967245, 0.6139807028410055 ], "peak_layer": 1, "peak_normalized": 1.7359303646705315, "population_cv": 0.3220412433027962, "rest_mean": 4.907019663530971e-05, "spike_contrast": 0.7193528234378936, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.529878449626267e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 8.144709863699973e-05, 7.730211655143648e-05, 7.289439963642508e-05, 7.000731420703232e-05, 6.740137905580923e-05, 6.305056740529835e-05, 5.999663699185476e-05, 5.856973439222202e-05, 5.5890508519951254e-05, 5.4439839004771784e-05, 5.1588234782684594e-05, 4.9732483603293076e-05, 4.965837433701381e-05, 4.774856279254891e-05, 4.576848732540384e-05, 4.519004141911864e-05, 4.3363412260077894e-05, 4.106453343410976e-05, 3.966558142565191e-05, 3.862750600092113e-05, 3.6894074582960457e-05, 3.647751145763323e-05, 3.517654840834439e-05, 3.4665710700210184e-05, 3.328007733216509e-05, 3.241290323785506e-05, 3.136717350571416e-05, 3.0930281354812905e-05, 2.9626178729813546e-05, 2.925785884144716e-05, 2.9087101211189292e-05, 2.8807000489905477e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 0.0001734125216898974, "first_to_last_ratio": 1.495517223041696, "imbalance_abs_log_ratio": 0.4024721155990803, "last_quartile_mean": 0.00011595488103921525, "mean": 0.00013990532806928968, "normalized": [ 1.3768889945483427, 1.3340940553978193, 1.2910049686488183, 1.2527203273458163, 1.2147536525239127, 1.1769854337196524, 1.14744939519443, 1.1220955988146075, 1.0944022472959734, 1.0773022658134643, 1.0508559212107247, 1.03206209473486, 1.0208171898483585, 1.000561882500425, 0.9805161605170362, 0.9663129296065233, 0.9487424991891081, 0.931931984738623, 0.9181974345490227, 0.9075104531838204, 0.8943640941785878, 0.8862664021756573, 0.8757830774574916, 0.86790401991425, 0.8562894022841879, 0.8478155487572495, 0.8387227683262297, 0.8314011142413483, 0.8224071457687988, 0.8161875050523302, 0.8112199678685613, 0.8064334645939688 ], "peak_layer": 1, "peak_normalized": 1.3768889945483427, "population_cv": 0.16325957486062354, "rest_mean": 0.0001431148236776978, "spike_contrast": 0.8564734849544942, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001225740517838858, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0001926341064972803, 0.00018664686649572104, 0.00018061847367789596, 0.00017526224837638438, 0.0001699505082797259, 0.00016466653323732316, 0.00016053428407758474, 0.00015698715287726372, 0.00015311270544771105, 0.00015072032692842185, 0.00014702034241054207, 0.00014439098595175892, 0.00014281776384450495, 0.00013998393842484802, 0.00013717943511437625, 0.00013519232743419707, 0.00013273413060232997, 0.00013038225006312132, 0.00012846071331296116, 0.00012696554767899215, 0.0001251263020094484, 0.00012399339175317436, 0.0001225267187692225, 0.00012142439663875848, 0.00011979944974882528, 0.00011861391249112785, 0.00011734178406186402, 0.00011631744564510882, 0.00011505914153531194, 0.00011418898066040128, 0.00011349399574100971, 0.00011282433843007311 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 0.00014675117381557357, "first_to_last_ratio": 1.2676837465168667, "imbalance_abs_log_ratio": 0.23719141364257146, "last_quartile_mean": 0.00011576323686313117, "mean": 0.00013042020850662084, "normalized": [ 1.169988069421413, 1.156175152925158, 1.1417665260225645, 1.1306726331060184, 1.1183621029407895, 1.1062694829504214, 1.0940115056230622, 1.0844989956600726, 1.0711591662817939, 1.0579765491085218, 1.0443900226505625, 1.035471661013805, 1.0260498632741444, 1.013992390086173, 1.0062479316494695, 0.9977001177294099, 0.9863163473759529, 0.9769959732660659, 0.9687585669786032, 0.9602628595878113, 0.9503845997255937, 0.9426104617668664, 0.9330243108845692, 0.9259760938667744, 0.914797625470175, 0.906289421438531, 0.8956357003805738, 0.888470885240542, 0.8806216558595509, 0.8756991509238615, 0.8712858841028995, 0.8681382926882502 ], "peak_layer": 1, "peak_normalized": 1.169988069421413, "population_cv": 0.09123006364857297, "rest_mean": 0.00013202972385678786, "spike_contrast": 0.9219804606102014, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012172882561571896, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00015259008796419948, 0.00015078860451467335, 0.00014890942838974297, 0.0001474625605624169, 0.00014585701865144074, 0.00014427989663090557, 0.00014268120867200196, 0.00014144058513920754, 0.00013970080181024969, 0.0001379815221298486, 0.00013620956451632082, 0.00013504642993211746, 0.0001338176371064037, 0.00013224509893916547, 0.00013123506505507976, 0.0001301202573813498, 0.00012863558367826045, 0.00012742001854348928, 0.00012634569429792464, 0.00012523768236860633, 0.00012394935765769333, 0.00012293545296415687, 0.00012168522516731173, 0.00012076599523425102, 0.00011930809705518186, 0.00011819845531135798, 0.00011680899478960782, 0.00011587455810513347, 0.00011485085997264832, 0.00011420886585256085, 0.00011363328667357564, 0.00011322277714498341 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00030002388848515693, "first_to_last_ratio": 2.2836575149168885, "imbalance_abs_log_ratio": 0.8257783308640597, "last_quartile_mean": 0.0001313786706305109, "mean": 0.0001947244400071213, "normalized": [ 1.8998833386658438, 1.7835645331795598, 1.6713073472446418, 1.5703972875779748, 1.4780250735507643, 1.3761669788219328, 1.3066447563403325, 1.2401013786965793, 1.1762106459027857, 1.150547637187856, 1.0768735950666006, 1.0264900078881174, 1.0140784159549234, 0.9666871251601796, 0.9096841069789114, 0.8754807117769676, 0.8421876125778792, 0.8073809902734768, 0.77911667677541, 0.7607651063686129, 0.7403822030704592, 0.7308855611321243, 0.7152791523187013, 0.7043381880055872, 0.6944157299089508, 0.686864629891504, 0.680599049276497, 0.6764772714226681, 0.6702412094778135, 0.6672258960044359, 0.6635863560708457, 0.6581114274310631 ], "peak_layer": 1, "peak_normalized": 1.8998833386658438, "population_cv": 0.3592853832280775, "rest_mean": 0.00020492727344390005, "spike_contrast": 0.6813594750078023, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013962913944851608, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0003699537192005664, 0.0003473036049399525, 0.0003254443872720003, 0.00030579473241232336, 0.00028780760476365685, 0.00026797334430739284, 0.0002544356684666127, 0.0002414780465187505, 0.0002290369593538344, 0.00022403974435292184, 0.00020969360775779933, 0.00019988269195891917, 0.0001974658516701311, 0.00018823760910890996, 0.00017713772831484675, 0.00017047749133780599, 0.00016399451124016196, 0.0001572168112033978, 0.00015171305858530104, 0.00014813955931458622, 0.00014417050988413393, 0.00014232128160074353, 0.00013928233238402754, 0.0001371518592350185, 0.00013521971413865685, 0.00013374933041632175, 0.00013252926873974502, 0.00013172665785532445, 0.00013051234418526292, 0.00012992518895771354, 0.00012921648158226162, 0.000128150379168801 ] } } }, "mlp_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "afe29f3ef4d2992353f0bb59fe7c5c2c5050fca671a53184825a8d63cd83a2bc", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.0027073738165199757, 0.002722575329244137, 0.0026850115973502398, 0.0026861224323511124, 0.00271838391199708, 0.0026645169127732515, 0.0026859452482312918, 0.0027359756641089916, 0.0026767742820084095, 0.0027451682835817337, 0.0026999011170119047, 0.002663193503394723, 0.002626648638397455, 0.0026531771291047335, 0.0026517712976783514, 0.002676903735846281, 0.002670086920261383, 0.0026355870068073273, 0.0026610493659973145, 0.0026388936676084995, 0.0026469926815479994, 0.00264862016774714, 0.002641494618728757, 0.002674198243767023, 0.002557571977376938, 0.00252733426168561, 0.0025468147359788418, 0.0026856842450797558, 0.002551089273765683, 0.0025778315030038357, 0.0026001501828432083, 0.0025662274565547705 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.00015858857113926206, "first_to_last_ratio": 3.757867690304466, "imbalance_abs_log_ratio": 1.3238516928952826, "last_quartile_mean": 4.220174423608114e-05, "mean": 8.573221236929385e-05, "normalized": [ 2.5461641896458365, 2.3032502265274117, 2.0924715218207024, 1.8886722264368014, 1.7031663195050915, 1.537434754996766, 1.4223955235407997, 1.3049501874408618, 1.2055297161045848, 1.1692542365736316, 1.0769571962730058, 1.0144505847411098, 0.9913601784485884, 0.9317001386730132, 0.857757082878185, 0.8155634736467603, 0.7682612945963089, 0.7217806418748633, 0.678338533281248, 0.6511820384364709, 0.6192882306865346, 0.6053793146733001, 0.5848721334223365, 0.5718144052517429, 0.5462281883927541, 0.527515083157495, 0.5112993585545178, 0.496920907994256, 0.47722358366421574, 0.46552107689700334, 0.4601234027970099, 0.45317424906679377 ], "peak_layer": 1, "peak_normalized": 2.5461641896458365, "population_cv": 0.5686091737198813, "rest_mean": 9.231269224848668e-05, "spike_contrast": 0.5437781067692293, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.0197621021652594e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00021828828903380781, 0.00019746273756027222, 0.00017939221288543195, 0.0001619200484128669, 0.00014601621660403907, 0.000131807682919316, 0.00012194511509733275, 0.00011187626660102978, 0.00010335272963857278, 0.00010024275252362713, 9.232992306351662e-05, 8.697109296917915e-05, 8.499150135321543e-05, 7.987671415321529e-05, 7.353741239057854e-05, 6.992006092332304e-05, 6.586474046343938e-05, 6.187985127326101e-05, 5.8155463193543255e-05, 5.582727681030519e-05, 5.309295011102222e-05, 5.1900507969548926e-05, 5.0142381951445714e-05, 4.902291402686387e-05, 4.682935104938224e-05, 4.522503513726406e-05, 4.3834825191879645e-05, 4.260212881490588e-05, 4.0913433622336015e-05, 3.991015182691626e-05, 3.944739728467539e-05, 3.885163096128963e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.00021242323236947414, "first_to_last_ratio": 1.8345651540958352, "imbalance_abs_log_ratio": 0.6068074801791611, "last_quartile_mean": 0.0001157894184871111, "mean": 0.00015111673860701558, "normalized": [ 1.771707915810251, 1.6366668487510905, 1.5242909365839221, 1.4186705636795862, 1.3264449439106374, 1.2453857912414694, 1.1893415085703798, 1.1330085309323608, 1.0858463851390703, 1.062861239127963, 1.023237806768965, 0.9942126959777651, 0.978436257513136, 0.9531027445378829, 0.9232414968561398, 0.904013237249231, 0.8853704564278231, 0.8680164030054869, 0.8515402788738432, 0.8395817783841374, 0.8267435189825344, 0.8185143642831039, 0.809172703595369, 0.8007918347797284, 0.7909896391756025, 0.7821052393452608, 0.7753388667989797, 0.7682453290925442, 0.7604995795745633, 0.7549901567959042, 0.7508572350862155, 0.7467737131490535 ], "peak_layer": 1, "peak_normalized": 1.771707915810251, "population_cv": 0.2714170004285523, "rest_mean": 0.00015645500981783562, "spike_contrast": 0.7816309251520459, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012229007406858728, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0002677347220014781, 0.00024732775636948645, 0.00023034587502479553, 0.00021438486874103546, 0.00020044803386554122, 0.0001881986390799284, 0.00017972940986510366, 0.00017121655400842428, 0.00016408956435043365, 0.00016061612404882908, 0.0001546283601783216, 0.00015024218009784818, 0.00014785809617023915, 0.0001440297783119604, 0.00013951724395155907, 0.000136611532070674, 0.0001337942958343774, 0.00013117180787958205, 0.0001286819897359237, 0.00012687486014328897, 0.00012493478425312787, 0.00012369122123345733, 0.00012227953993715346, 0.00012101305037504062, 0.00011953177454415709, 0.00011818919301731512, 0.00011716668086592108, 0.00011609472858253866, 0.00011492421617731452, 0.00011409165017539635, 0.00011346709652571008, 0.00011285000800853595 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.18837904930114746, "first_to_last_ratio": 1.8345651581276443, "imbalance_abs_log_ratio": 0.6068074823768531, "last_quartile_mean": 0.10268321540206671, "mean": 0.13401183625683188, "normalized": [ 1.7717078885231814, 1.6366668119471057, 1.524290957183596, 1.418670514273678, 1.326444920903592, 1.2453858718827995, 1.1893415500975306, 1.1330085716119305, 1.0858463365841577, 1.062861212873729, 1.0232377434953237, 0.9942127340740732, 0.9784362443707608, 0.9531027271661117, 0.9232414917376033, 0.904013241653558, 0.885370477646398, 0.8680164390463408, 0.8515402681847718, 0.8395816969955481, 0.8267435345782925, 0.8185144279720658, 0.8091727254708025, 0.8007918405918669, 0.7909896328038464, 0.7821052665924377, 0.7753388466443417, 0.7682453528580236, 0.7604996016741158, 0.7549901615676585, 0.7508572336421532, 0.7467736753526057 ], "peak_layer": 1, "peak_normalized": 1.7717078885231814, "population_cv": 0.27141699472708714, "rest_mean": 0.1387458680956452, "spike_contrast": 0.7816309473985974, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.10844806432724, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.23742982745170593, 0.2193327248096466, 0.2042730301618576, 0.19011864066123962, 0.1777593195438385, 0.16689644753932953, 0.15938584506511688, 0.151836559176445, 0.1455162614583969, 0.1424359828233719, 0.13712596893310547, 0.13323627412319183, 0.13112203776836395, 0.12772704660892487, 0.1237252876162529, 0.1211484745144844, 0.11865012347698212, 0.11632447689771652, 0.11411647498607635, 0.11251388490200043, 0.11079341918230057, 0.10969062149524689, 0.10843872278928757, 0.10731558501720428, 0.10600197315216064, 0.10481136292219162, 0.10390458256006241, 0.1029539704322815, 0.10191594809293747, 0.10117761790752411, 0.10062375664710999, 0.10007651150226593 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 6.873187476230669e-05, "first_to_last_ratio": 2.248757500145603, "imbalance_abs_log_ratio": 0.8103778415280068, "last_quartile_mean": 3.0564378221242805e-05, "mean": 4.685761763312257e-05, "normalized": [ 1.7400006587426948, 1.6405194725375354, 1.5551306973978087, 1.4944078341288562, 1.4327724019957897, 1.3414187773842097, 1.2804088050749736, 1.2499327782691996, 1.1927955171655855, 1.1608931065609767, 1.0981756972550076, 1.0577423904261343, 1.0542786038595535, 1.0218640087393278, 0.9775807389332813, 0.9570972402588515, 0.9209982861589985, 0.8815092249082567, 0.8478232134430647, 0.8204952153707635, 0.7871780668180614, 0.7759097745574741, 0.7506963524238334, 0.7421155343193898, 0.7072927325438048, 0.6891814936654451, 0.6695399899996352, 0.6616343983211914, 0.6345111343049281, 0.6212687927046295, 0.6200231913288072, 0.6148038704019305 ], "peak_layer": 1, "peak_normalized": 1.7400006587426948, "population_cv": 0.3217536161653951, "rest_mean": 4.900405669845727e-05, "spike_contrast": 0.719671983430414, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.526684668031521e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 8.153228554874659e-05, 7.687083416385576e-05, 7.286971958819777e-05, 7.002439087955281e-05, 6.71363013680093e-05, 6.285568815656006e-05, 5.999690620228648e-05, 5.8568872191244736e-05, 5.5891556257847697e-05, 5.439668530016206e-05, 5.145789691596292e-05, 4.9563288484932855e-05, 4.9400983698433265e-05, 4.7882112994557247e-05, 4.580710447044112e-05, 4.484729652176611e-05, 4.3155785533599555e-05, 4.130542220082134e-05, 3.972697595600039e-05, 3.844645107164979e-05, 3.688528886414133e-05, 3.635728353401646e-05, 3.517584264045581e-05, 3.477376594673842e-05, 3.314205241622403e-05, 3.229340290999971e-05, 3.1373048841487616e-05, 3.10026116494555e-05, 2.9731680115219206e-05, 2.9111175535945222e-05, 2.9052809622953646e-05, 2.8808244678657502e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 0.0001719075116852764, "first_to_last_ratio": 1.4894635014227988, "imbalance_abs_log_ratio": 0.39841598895878805, "last_quartile_mean": 0.00011541572621354135, "mean": 0.00013894635094402474, "normalized": [ 1.3738985779819257, 1.3296082759737424, 1.2883384004275333, 1.2499940655466777, 1.2130730176151365, 1.1755503979222366, 1.147409009170091, 1.11990595257071, 1.0929037170303153, 1.075598370036652, 1.0512762916941902, 1.0312594730951, 1.0194418974669357, 1.0018996501730997, 0.9803460168062212, 0.9648004557293595, 0.9493010807852901, 0.9344788929945884, 0.9198953859218769, 0.908001147597105, 0.8952913731815082, 0.886953676542665, 0.8771894463102521, 0.8683888934041172, 0.8579399877904575, 0.8482799643655919, 0.8408636339472488, 0.8330870307470335, 0.82434015220157, 0.8181187963300758, 0.8135579418994486, 0.809009026741244 ], "peak_layer": 1, "peak_normalized": 1.3738985779819257, "population_cv": 0.16161423472023242, "rest_mean": 0.0001421073115866252, "spike_contrast": 0.8576417505420824, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012187716347398237, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0001908981939777732, 0.0001847442181315273, 0.00017900991952046752, 0.00017368211410939693, 0.00016855206922627985, 0.00016333843814209104, 0.00015942829486448318, 0.00015560684551019222, 0.0001518549834145233, 0.00014945046859793365, 0.00014607100456487387, 0.0001432897406630218, 0.00014164773165248334, 0.00013921030040364712, 0.000136215501697734, 0.0001340555027127266, 0.0001319019211223349, 0.00012984243221580982, 0.0001278161071240902, 0.00012616344611160457, 0.00012439746933523566, 0.00012323897681199014, 0.00012188227265141904, 0.00012065946793882176, 0.00011920763063244522, 0.00011786540562752634, 0.00011683493357850239, 0.00011575440294109285, 0.0001145390560850501, 0.00011367462138878182, 0.00011304090730845928, 0.00011240885214647278 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 0.00014552221000485588, "first_to_last_ratio": 1.262697055715921, "imbalance_abs_log_ratio": 0.2332499537229619, "last_quartile_mean": 0.00011524712863320019, "mean": 0.00012953460168318998, "normalized": [ 1.1676213631634658, 1.1525055688027706, 1.1391580058081985, 1.1289307982409096, 1.1166731577821778, 1.1054001772932092, 1.094813144753513, 1.0822854389705017, 1.0679308603491235, 1.0564894822224282, 1.0456838360716472, 1.0337072702557388, 1.0234945545465333, 1.0158819477934218, 1.0051772952892828, 0.9951467950359763, 0.9866643397779161, 0.9787559418439278, 0.970448736962688, 0.9607908684400951, 0.9509939230158972, 0.9428061355388626, 0.934924980051271, 0.9261034265480621, 0.9170582603044295, 0.9058562786860701, 0.8981778407016658, 0.8907564927732574, 0.8831168120837269, 0.8778216107141017, 0.8745192645108896, 0.8703053916682415 ], "peak_layer": 1, "peak_normalized": 1.1676213631634658, "population_cv": 0.08967236486753827, "rest_mean": 0.00013110875065817877, "spike_contrast": 0.9231588022206524, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012103419721825049, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00015124736819416285, 0.00014928934979252517, 0.000147560378536582, 0.00014623560127802193, 0.00014464781270362437, 0.00014318757166620344, 0.00014181618462316692, 0.00014019341324456036, 0.0001383339986205101, 0.00013685194426216185, 0.00013545223919209093, 0.00013390085950959474, 0.00013257795944809914, 0.00013159186346456409, 0.00013020524056628346, 0.00012890594371128827, 0.00012780717224813998, 0.00012678276107180864, 0.0001257066905964166, 0.0001244556624442339, 0.00012318661902099848, 0.0001221260172314942, 0.00012110513489460573, 0.0001199624384753406, 0.00011879077646881342, 0.00011733973224181682, 0.00011634510883595794, 0.00011538378748809919, 0.0001143941844929941, 0.00011370827269274741, 0.00011328050459269434, 0.0001127346622524783 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00029741139769612346, "first_to_last_ratio": 2.278072415841521, "imbalance_abs_log_ratio": 0.8233296537390045, "last_quartile_mean": 0.00013055396993877366, "mean": 0.00019328556300024502, "normalized": [ 1.889398576482247, 1.780233894901915, 1.6772594716902607, 1.566224958959574, 1.4762232610465842, 1.375571144634086, 1.3083686481730985, 1.2364401527267181, 1.173148897934763, 1.1508864184295666, 1.080457040307468, 1.0318582947152708, 1.0081957004611959, 0.9698994715426978, 0.9066127339785177, 0.8754550048258108, 0.8446146862114313, 0.8115724449181009, 0.7806677558079268, 0.7613480996239201, 0.7403134026279838, 0.7294052762839086, 0.7160186976339193, 0.7062575707090604, 0.6942777320753531, 0.6866005528973866, 0.6829938476675228, 0.6753403086482458, 0.6715967314167978, 0.6686860556999406, 0.6640286432895989, 0.6600445236791291 ], "peak_layer": 1, "peak_normalized": 1.889398576482247, "population_cv": 0.3579154260008552, "rest_mean": 0.00020340604748559632, "spike_contrast": 0.681567477924495, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013863494677934796, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0003651934675872326, 0.0003440935106482357, 0.00032419004128314555, 0.0003027286729775369, 0.0002853326441254467, 0.00026587804313749075, 0.00025288877077400684, 0.00023898603103589267, 0.00022675274522043765, 0.00022244972933549434, 0.0002088367473334074, 0.00019944331143051386, 0.00019486967357806861, 0.00018746756541077048, 0.00017523515271022916, 0.00016921281348913908, 0.0001632518251426518, 0.00015686523693148047, 0.00015089180669747293, 0.00014715759607497603, 0.00014309189282357693, 0.00014098350948188454, 0.00013839607709087431, 0.0001365093921776861, 0.0001341938623227179, 0.00013270997442305088, 0.00013201285037212074, 0.00013053353177383542, 0.0001298099523410201, 0.00012924736074637622, 0.00012834715016651899, 0.00012757707736454904 ] } } }, "post_attention_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "31853ffb07b24cc59e774bb20656840b4358b3080b42516d380f2dc0f46404c8", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.0022265110164880753, 0.0040087271481752396, 0.005379301495850086, 0.006410556845366955, 0.0027253651060163975, 0.004838948138058186, 0.006175771821290255, 0.00742313964292407, 0.0030937574338167906, 0.005150026176124811, 0.007019025273621082, 0.008143599145114422, 0.004255065228790045, 0.0066224560141563416, 0.008129785768687725, 0.010099578648805618, 0.0041999719105660915, 0.00719039561226964, 0.009030314162373543, 0.010736005380749702, 0.005131533369421959, 0.00787449348717928, 0.009912566281855106, 0.012113703414797783, 0.0059690922498703, 0.00876442901790142, 0.010806876234710217, 0.012353328987956047, 0.006502739619463682, 0.009470963850617409, 0.012123330496251583, 0.01384514570236206 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.00015962236011546338, "first_to_last_ratio": 3.7698422272252903, "imbalance_abs_log_ratio": 1.3270331510418396, "last_quartile_mean": 4.2341920561739244e-05, "mean": 8.61727822893954e-05, "normalized": [ 2.542734798705971, 2.3221448837042376, 2.101145461274622, 1.8911858524935854, 1.7004204056817789, 1.5354142426015858, 1.4229001770142031, 1.3028733810328659, 1.2049412416934622, 1.1705258152816296, 1.0773908232649334, 1.0133220774376726, 0.990609360100461, 0.9272534949358001, 0.8550385909364457, 0.8185651575045715, 0.7703972051466339, 0.7173947009730917, 0.6760256695667757, 0.6514523109028573, 0.618419077603279, 0.6053620388084041, 0.5838930841859182, 0.5697043695032278, 0.5457802700901755, 0.5276149005446632, 0.5104701332743111, 0.49517988582787786, 0.47527562953524777, 0.46579059316028215, 0.45896435764104826, 0.45181000957238127 ], "peak_layer": 1, "peak_normalized": 2.542734798705971, "population_cv": 0.5705202801130907, "rest_mean": 9.280119640472414e-05, "spike_contrast": 0.542873885449774, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.03793460666202e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00021911453222855926, 0.00020010568550787866, 0.00018106155039276928, 0.00016296874673571438, 0.00014652995741926134, 0.00013231091725174338, 0.0001226152671733871, 0.00011227222421439365, 0.00010383313929196447, 0.00010086746624438092, 9.284176485380158e-05, 8.732078276807442e-05, 8.536356472177431e-05, 7.99040135461837e-05, 7.368105434579775e-05, 7.053803710732609e-05, 6.638727063545957e-05, 6.181989738252014e-05, 5.825501284562051e-05, 5.613745815935545e-05, 5.329089253791608e-05, 5.216573117650114e-05, 5.0315691623836756e-05, 4.9093010602518916e-05, 4.703140439232811e-05, 4.546604395727627e-05, 4.398863165988587e-05, 4.267102849553339e-05, 4.095582335139625e-05, 4.013847137684934e-05, 3.955023566959426e-05, 3.893372559105046e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.00021442477373057045, "first_to_last_ratio": 1.8431254562763404, "imbalance_abs_log_ratio": 0.6114627481432037, "last_quartile_mean": 0.00011633759004325839, "mean": 0.00015221248349917005, "normalized": [ 1.7740805953138137, 1.6484457246979003, 1.5301434809496548, 1.4217241600875643, 1.3264958046529274, 1.2454577490392524, 1.1897861330408546, 1.1336265611690741, 1.086914920172498, 1.064301358569379, 1.0227209980571423, 0.9941753882589759, 0.9788979908211966, 0.9511107967643203, 0.9228143274453928, 0.9052447611165039, 0.8846530997418666, 0.8652930871753776, 0.8497381552743479, 0.8388656494639406, 0.8256334792987466, 0.8176309647342147, 0.8076951735882674, 0.8000661307359944, 0.7892605467734257, 0.78144613040264, 0.7731219607566042, 0.7664988488799842, 0.7584935618388989, 0.7529639051546407, 0.7485195296370439, 0.7441790263875565 ], "peak_layer": 1, "peak_normalized": 1.7740805953138137, "population_cv": 0.2736411578809574, "rest_mean": 0.00015762286892384981, "spike_contrast": 0.7803207938393821, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012299640220589936, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00027003721334040165, 0.00025091401766985655, 0.00023290693934541196, 0.0002164041652576998, 0.000201909220777452, 0.0001895742170745507, 0.00018110030214302242, 0.00017255211423616856, 0.00016544201935175806, 0.00016199995297938585, 0.00015567090304102749, 0.00015132590488065034, 0.0001490004942752421, 0.00014477093645837158, 0.00014046386058907956, 0.00013778955326415598, 0.0001346552453469485, 0.00013170840975362808, 0.00012934075493831187, 0.00012768582382705063, 0.00012567172234412283, 0.00012445363972801715, 0.00012294128828216344, 0.00012178005272289738, 0.00012013530795229599, 0.00011894585622940212, 0.00011767881369451061, 0.00011667069338727742, 0.00011545218876563013, 0.00011461050598882139, 0.00011393401655368507, 0.00011327333777444437 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.190154030919075, "first_to_last_ratio": 1.84312545379361, "imbalance_abs_log_ratio": 0.6114627467961817, "last_quartile_mean": 0.10316933691501617, "mean": 0.13498355145566165, "normalized": [ 1.774080628234443, 1.6484456620184311, 1.5301434020504485, 1.4217242444591272, 1.326495766996536, 1.2454576834375484, 1.189786122599316, 1.133626627255238, 1.0869149569673349, 1.0643014004698326, 1.0227209907477024, 0.9941753870542451, 0.9788979589698585, 0.9511107512164827, 0.9228143032197887, 0.9052447972745712, 0.8846531331847195, 0.8652931173486506, 0.8497382179112226, 0.8388656682549475, 0.8256334805754935, 0.8176309131099365, 0.8076951535479161, 0.8000661540388709, 0.7892605573866218, 0.7814461538045975, 0.7731219580171443, 0.7664988542586116, 0.7584935821787124, 0.7529638601652359, 0.7485195164462104, 0.7441789968002049 ], "peak_layer": 1, "peak_normalized": 1.774080628234443, "population_cv": 0.27364115345546325, "rest_mean": 0.13978153549962574, "spike_contrast": 0.7803207857775153, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.10907443761825561, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.2394717037677765, 0.22251304984092712, 0.2065441906452179, 0.19190938770771027, 0.1790551096200943, 0.16811630129814148, 0.16060155630111694, 0.1530209481716156, 0.14671564102172852, 0.1436631828546524, 0.13805051147937775, 0.13419732451438904, 0.13213512301445007, 0.1283843070268631, 0.1245647519826889, 0.12219315767288208, 0.11941362172365189, 0.11680033802986145, 0.11470068246126175, 0.1132330670952797, 0.11144693940877914, 0.11036672443151474, 0.10902556031942368, 0.10799577087163925, 0.10653719305992126, 0.10548237711191177, 0.1043587476015091, 0.10346473753452301, 0.10238415747880936, 0.10163773596286774, 0.10103782266378403, 0.10045192390680313 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 6.883367768750759e-05, "first_to_last_ratio": 2.249759357551583, "imbalance_abs_log_ratio": 0.810823258297238, "last_quartile_mean": 3.059601795030176e-05, "mean": 4.691846504556452e-05, "normalized": [ 1.7359059710084281, 1.6475789727938472, 1.5536456441204052, 1.4921194549590373, 1.4365657321885223, 1.3438513058553838, 1.2787343947807137, 1.2483299149152907, 1.1912354282513808, 1.1603083401306369, 1.099547144002948, 1.059972440022979, 1.0583955432739494, 1.0176800955770247, 0.9754963047746087, 0.9631595708780456, 0.9242400656751482, 0.8752301116137658, 0.8454245377074081, 0.8232853322610043, 0.7863494898171388, 0.7774642190168813, 0.7497486129019063, 0.7388483538626245, 0.709315327139865, 0.6908343794807076, 0.6685469983108738, 0.6592365073812129, 0.6314341746588822, 0.6235898169926835, 0.6199474545603838, 0.6139783610863121 ], "peak_layer": 1, "peak_normalized": 1.7359059710084281, "population_cv": 0.3220406003070692, "rest_mean": 4.9070239684725595e-05, "spike_contrast": 0.7193541792517962, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.52988819940947e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 8.144604362314567e-05, 7.730187644483522e-05, 7.28946688468568e-05, 7.000795449130237e-05, 6.740145909134299e-05, 6.305144052021205e-05, 5.999625500408001e-05, 5.8569723478285596e-05, 5.589093780145049e-05, 5.4439886298496276e-05, 5.158906424185261e-05, 4.973227987647988e-05, 4.9658294301480055e-05, 4.774798799189739e-05, 4.576878927764483e-05, 4.51899686595425e-05, 4.3363925215089694e-05, 4.1064453398576006e-05, 3.9666021621087566e-05, 3.86272840842139e-05, 3.689431105158292e-05, 3.6477427784120664e-05, 3.5177054087398574e-05, 3.466563066467643e-05, 3.3279986382694915e-05, 3.2412888685939834e-05, 3.136719897156581e-05, 3.093036502832547e-05, 2.9625922252307646e-05, 2.9257877031341195e-05, 2.9086982976878062e-05, 2.8806922273361124e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 0.00017341253987979144, "first_to_last_ratio": 1.4955148931299893, "imbalance_abs_log_ratio": 0.4024705576674927, "last_quartile_mean": 0.00011595507385209203, "mean": 0.00013990553861731314, "normalized": [ 1.3768818258178377, 1.3340879911950096, 1.2910054180628048, 1.2527236427111486, 1.2147542166907819, 1.1769889670731195, 1.1474411155803916, 1.1220953663134672, 1.0944075691302455, 1.0773007485615413, 1.0508678613623372, 1.0320596054419082, 1.0208161736522385, 1.000559648637946, 0.9805197815181176, 0.9663128275349431, 0.948744191771126, 0.9319305822463463, 0.9182019814348666, 0.9075038868226692, 0.8943650364968407, 0.886262676120959, 0.8757907045344768, 0.8679002174812178, 0.856288373659771, 0.8478178092769747, 0.8387215581135022, 0.8313975747674917, 0.8224084044020752, 0.8161864847722404, 0.8112200992010112, 0.8064376596145924 ], "peak_layer": 1, "peak_normalized": 1.3768818258178377, "population_cv": 0.16325888721442564, "rest_mean": 0.0001431150389921472, "spike_contrast": 0.8564734877369171, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001225742365932092, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0001926333934534341, 0.00018664629897102714, 0.00018061880837194622, 0.0001752629759721458, 0.00016995084297377616, 0.0001646672753849998, 0.00016053336730692536, 0.00015698735660407692, 0.00015311368042603135, 0.00015072034148033708, 0.00014702223415952176, 0.00014439085498452187, 0.0001428178366040811, 0.00013998383656144142, 0.00013718014815822244, 0.00013519251660909504, 0.00013273456715978682, 0.00013038225006312132, 0.00012846154277212918, 0.00012696482008323073, 0.00012512662215158343, 0.0001239930570591241, 0.00012252797023393214, 0.000121424047392793, 0.00011979948612861335, 0.00011861440725624561, 0.00011734179133782163, 0.0001163171255029738, 0.00011505949078127742, 0.00011418900976423174, 0.00011349418491590768, 0.00011282509512966499 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 0.00014675183047074825, "first_to_last_ratio": 1.2676545313577883, "imbalance_abs_log_ratio": 0.23716836728277432, "last_quartile_mean": 0.00011576642282307148, "mean": 0.00013041823922321782, "normalized": [ 1.1700698937843759, 1.1560440994564782, 1.1417398043557874, 1.130463200949706, 1.118403537277814, 1.1065304334184665, 1.0939881912841023, 1.0846815122144253, 1.0709468270677047, 1.0579907390333845, 1.0442838370166418, 1.035550673160169, 1.025959021745254, 1.0140203095307772, 1.0062047700154308, 0.9976631870190885, 0.9861765922450596, 0.9770016877798172, 0.9687702939454803, 0.9602422119882037, 0.9504995944091904, 0.9425883202531964, 0.9330014667281973, 0.9259385264566372, 0.9147443798098084, 0.906202796813466, 0.8958065504175365, 0.8885655303249116, 0.8806194435723184, 0.8757905905296257, 0.8713783728577705, 0.8681336045391749 ], "peak_layer": 1, "peak_normalized": 1.1700698937843759, "population_cv": 0.09121429133712369, "rest_mean": 0.0001320278307604086, "spike_contrast": 0.921975648780332, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001217264449223876, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0001525984553154558, 0.0001507692359155044, 0.000148903694935143, 0.00014743302017450333, 0.00014586022007279098, 0.00014431175077334046, 0.00014267601363826543, 0.00014146225294098258, 0.000139670999487862, 0.00013798128929920495, 0.00013619365927297622, 0.00013505469541996717, 0.00013380376913119107, 0.00013224674330558628, 0.00013122745440341532, 0.00013011347618885338, 0.00012861541472375393, 0.00012741883983835578, 0.0001263453159481287, 0.0001252330985153094, 0.0001239624834852293, 0.00012293070903979242, 0.00012168040848337114, 0.00012075927224941552, 0.00011929935135412961, 0.00011818537313956767, 0.00011682951299007982, 0.00011588515189941972, 0.00011484883725643158, 0.00011421906674513593, 0.00011364363308530301, 0.00011322045611450449 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00030004180916876066, "first_to_last_ratio": 2.2837506327398005, "imbalance_abs_log_ratio": 0.8258191057718037, "last_quartile_mean": 0.00013138116082700435, "mean": 0.00019473525253488333, "normalized": [ 1.9000096511818072, 1.783700741635724, 1.6708066910504589, 1.570366884717585, 1.4783058799917985, 1.376185321596543, 1.3064973297630273, 1.2402700043336092, 1.1767953849206394, 1.1505341943958838, 1.0771542571420398, 1.026098162637436, 1.013898063759585, 0.9669473025647412, 0.9095398902231254, 0.8755205777387188, 0.842301214161826, 0.8073124727024382, 0.7791991818813049, 0.760574607765185, 0.7402055397447304, 0.7308143413325056, 0.7152667122155245, 0.7043714155678553, 0.6942467749765324, 0.6868749151283189, 0.680655564574882, 0.676363713518914, 0.6701061029149729, 0.6672276319278244, 0.6636177363814285, 0.6582317375530352 ], "peak_layer": 1, "peak_normalized": 1.9000096511818072, "population_cv": 0.35930746026536914, "rest_mean": 0.00020494150898971216, "spike_contrast": 0.6812747128050888, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013962146767880766, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0003699988592416048, 0.0003473494143690914, 0.0003253649629186839, 0.0003058057918678969, 0.0002878782688640058, 0.0002679917961359024, 0.0002544210874475539, 0.00024152429250534624, 0.00022916354646440595, 0.000224049566895701, 0.00020975990628357977, 0.00019981748482678086, 0.000197441695490852, 0.0001882987271528691, 0.00017711948021315038, 0.00017049472080543637, 0.00016402573965024203, 0.00015721219824627042, 0.00015173754945863038, 0.00014811068831477314, 0.00014414411270990968, 0.0001423153153154999, 0.0001392876438330859, 0.00013716594548895955, 0.00013519432104658335, 0.00013375876005738974, 0.00013254763325676322, 0.0001317118585575372, 0.00013049328117631376, 0.00012993274140171707, 0.00012922976748086512, 0.00012818092363886535 ] } } }, "post_mlp_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "7fd9c911e93175896ad618842c120e77b3f017c17b7eec9c2bfb01fc6dbc8ac2", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.0034885562490671873, 0.004831995815038681, 0.005952076520770788, 0.006925336085259914, 0.00385080324485898, 0.005542940925806761, 0.006729288958013058, 0.007927131839096546, 0.004081308841705322, 0.005811207927763462, 0.007534808944910765, 0.008581922389566898, 0.00499920267611742, 0.007102519739419222, 0.008542952127754688, 0.010447698645293713, 0.005041283555328846, 0.007636935915797949, 0.009387043304741383, 0.011020516976714134, 0.005807018838822842, 0.008367395959794521, 0.01024632342159748, 0.012378672137856483, 0.006453148555010557, 0.009127611294388771, 0.011037957854568958, 0.012611051090061665, 0.007010402157902718, 0.009794782847166061, 0.012445803731679916, 0.014087481424212456 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.00015858902861509705, "first_to_last_ratio": 3.757869257587208, "imbalance_abs_log_ratio": 1.3238521099622325, "last_quartile_mean": 4.220184837322449e-05, "mean": 8.57325238712292e-05, "normalized": [ 2.5462257183789148, 2.3032637538343064, 2.0924289533379263, 1.8886492391629932, 1.7031256747388024, 1.5374176267930122, 1.4224231993460965, 1.3049597038569594, 1.2055618292025094, 1.169245405308266, 1.0769812048488787, 1.0144334047898542, 0.9913512297333369, 0.9317090592952362, 0.8577499774884012, 0.81555804918997, 0.7682725064221458, 0.7217807351250844, 0.6783324192661732, 0.6511800118936032, 0.6192916242842709, 0.6053728292385717, 0.5848720876072855, 0.5718124973482859, 0.5462302773879167, 0.5275173250132701, 0.5112998770983732, 0.49691833866078555, 0.47721658788927634, 0.46551989467389665, 0.46012635628943105, 0.45317260249616614 ], "peak_layer": 1, "peak_normalized": 2.5462257183789148, "population_cv": 0.5686111305901927, "rest_mean": 9.231303704803361e-05, "spike_contrast": 0.5437775023084314, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.019775271648541e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00021829435718245804, 0.00019746461475733668, 0.0001793892151908949, 0.0001619186659809202, 0.00014601326256524771, 0.00013180669338908046, 0.00012194793089292943, 0.00011187748896190897, 0.00010335585830034688, 0.00010024235962191597, 9.233231685357168e-05, 8.69699360919185e-05, 8.499104296788573e-05, 7.987776916706935e-05, 7.353707042057067e-05, 6.991984992055222e-05, 6.586594099644572e-05, 6.188008410390466e-05, 5.8155150327365845e-05, 5.582730591413565e-05, 5.309343396220356e-05, 5.190014053368941e-05, 5.014256021240726e-05, 4.90229285787791e-05, 4.682970029534772e-05, 4.522539165918715e-05, 4.383502891869284e-05, 4.2602063331287354e-05, 4.0912982512963936e-05, 3.991019548266195e-05, 3.944779382436536e-05, 3.885163096128963e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.00021242368711682502, "first_to_last_ratio": 1.8345658968465297, "imbalance_abs_log_ratio": 0.6068078850437921, "last_quartile_mean": 0.0001157896194854402, "mean": 0.0001511171253696375, "normalized": [ 1.7717451736709768, 1.636674793183293, 1.5242755762025908, 1.4186515254985965, 1.3264266232459345, 1.2453738409548603, 1.1893512719369177, 1.1330135273991528, 1.085860457802012, 1.0628542818802846, 1.023252039669589, 0.9942070699734702, 0.9784308644775869, 0.9531083940389613, 0.9232390376562555, 0.9040072643238835, 0.8853775311228679, 0.8680181295617843, 0.8515367513400655, 0.8395795332993111, 0.8267439067367663, 0.8185133286645813, 0.8091730400239778, 0.800791325997512, 0.7909899739933552, 0.7821072339266707, 0.7753403490721149, 0.7682422073314642, 0.7604971517068638, 0.7549863467424895, 0.7508596466770359, 0.7467718018887739 ], "peak_layer": 1, "peak_normalized": 1.7717451736709768, "population_cv": 0.2714186609524326, "rest_mean": 0.00015645541295978345, "spike_contrast": 0.7816308178118705, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012229037238284946, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0002677410375326872, 0.0002473295899108052, 0.00023034414334688336, 0.00021438254043459892, 0.00020044577831868082, 0.00018819731485564262, 0.00017973134526982903, 0.000171217747265473, 0.0001640921109355986, 0.00016061548376455903, 0.00015463090676348656, 0.00015024171443656087, 0.0001478576596127823, 0.00014403100067283958, 0.00013951722939964384, 0.00013661097909789532, 0.00013379570737015456, 0.0001311724045081064, 0.0001286817860091105, 0.00012687484559137374, 0.0001249351626029238, 0.00012369138130452484, 0.00012227990373503417, 0.00012101328320568427, 0.00011953213106608018, 0.0001181897969217971, 0.0001171672047348693, 0.00011609455395955592, 0.00011492414341773838, 0.0001140913664130494, 0.00011346775136189535, 0.00011285000800853595 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.18837944976985455, "first_to_last_ratio": 1.8345658800697515, "imbalance_abs_log_ratio": 0.6068078758989703, "last_quartile_mean": 0.10268339328467846, "mean": 0.1340121782850474, "normalized": [ 1.7717451751532867, 1.6366747548098712, 1.5242756140248972, 1.4186514377480737, 1.3264266357207033, 1.2453739091726768, 1.1893513017871604, 1.1330134634090196, 1.0858604665419038, 1.0628542749024097, 1.0232520332438115, 0.9942070832344971, 0.9784308561813967, 0.9531084116998004, 0.9232390798282929, 0.9040072094634526, 0.8853775581660035, 0.8680181154203164, 0.8515367049614931, 0.8395794986042229, 0.8267439263823189, 0.8185133396785964, 0.809173050926764, 0.8007913534932437, 0.7909900046670136, 0.782107217826699, 0.7753403703776178, 0.7682422246066585, 0.7604971603471317, 0.7549863443945886, 0.7508596538022533, 0.7467717694238248 ], "peak_layer": 1, "peak_normalized": 1.7717451751532867, "population_cv": 0.27141865758333467, "rest_mean": 0.13874622461972413, "spike_contrast": 0.7816308398662988, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.10844832807779312, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.23743543028831482, 0.21933434903621674, 0.20427149534225464, 0.19011656939983368, 0.17775732278823853, 0.16689527034759521, 0.15938755869865417, 0.15183760225772858, 0.14551852643489838, 0.14243541657924652, 0.13712823390960693, 0.1332358568906784, 0.1311216503381729, 0.12772813439369202, 0.1237252801656723, 0.12114797532558441, 0.1186513751745224, 0.1163249984383583, 0.11411628872156143, 0.11251387745141983, 0.11079375445842743, 0.10969075560569763, 0.10843904316425323, 0.107315793633461, 0.10600229352712631, 0.104811891913414, 0.10390505194664001, 0.10295381397008896, 0.1019158810377121, 0.10117736458778381, 0.10062433779239655, 0.10007651150226593 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 6.873207303215167e-05, "first_to_last_ratio": 2.248757814140914, "imbalance_abs_log_ratio": 0.8103779811585758, "last_quartile_mean": 3.0564462122129044e-05, "mean": 4.6857780716891284e-05, "normalized": [ 1.7400119939228862, 1.6405409364325143, 1.555121247708426, 1.4944054279879166, 1.4327475398601994, 1.3414100714975594, 1.2804101716466747, 1.2499370459040815, 1.192804641985012, 1.1608858830017554, 1.098191983597808, 1.057744066133373, 1.0542717513613309, 1.021869458338179, 0.9775749297615711, 0.9570927445976433, 0.9210091333285576, 0.8815125232807631, 0.847820728512489, 0.8204970180441978, 0.7871838673851995, 0.7759041238100245, 0.7506938949785397, 0.7421070509174572, 0.7072987335050381, 0.6891811912847667, 0.6695407652827761, 0.6616333377893114, 0.6345027924930273, 0.6212629426015038, 0.6200302724102036, 0.614801730639213 ], "peak_layer": 1, "peak_normalized": 1.7400119939228862, "population_cv": 0.32175451551289735, "rest_mean": 4.9004235565748616e-05, "spike_contrast": 0.7196709452950106, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.5266924533061685e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 8.153310045599937e-05, 7.687210745643824e-05, 7.28695304132998e-05, 7.002452184678987e-05, 6.713537004543468e-05, 6.28554989816621e-05, 5.9997179050697014e-05, 5.8569276006892323e-05, 5.589217835222371e-05, 5.439653614303097e-05, 5.1458839152473956e-05, 4.9563539505470544e-05, 4.9400834541302174e-05, 4.788253500009887e-05, 4.58069916930981e-05, 4.4847241952084005e-05, 4.315644400776364e-05, 4.130572051508352e-05, 3.972699778387323e-05, 3.8446669350378215e-05, 3.688568904181011e-05, 3.635714529082179e-05, 3.5175849916413426e-05, 3.477348946034908e-05, 3.3142448955914006e-05, 3.2293501135427505e-05, 3.13731943606399e-05, 3.1002669857116416e-05, 2.9731392714893445e-05, 2.9111002731951885e-05, 2.905324254243169e-05, 2.8808244678657502e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 0.00017190765356644988, "first_to_last_ratio": 1.4894621368061003, "imbalance_abs_log_ratio": 0.3984150727783497, "last_quartile_mean": 0.00011541592721187044, "mean": 0.00013894661265112518, "normalized": [ 1.3739040544627474, 1.3296102750467884, 1.2883342981472405, 1.2499918159027226, 1.2130627732820631, 1.175544413473833, 1.1474083142393565, 1.1199112790671786, 1.092909618037032, 1.0755929927697745, 1.0512861461262244, 1.0312571117882559, 1.0194388253067554, 1.0019061415077835, 0.980344379776871, 0.9647961249933756, 0.9493075664595542, 0.9344817410273759, 0.9198929201794295, 0.9079992279088673, 0.8952920956888009, 0.8869530532606651, 0.8771902552778059, 0.8683889858357474, 0.8579407806493952, 0.8482823463732029, 0.8408658204630546, 0.8330843095877946, 0.8243382853592501, 0.818115422616924, 0.8135611224181157, 0.8090075029660142 ], "peak_layer": 1, "peak_normalized": 1.3739040544627474, "population_cv": 0.16161403917150835, "rest_mean": 0.00014210757001786045, "spike_contrast": 0.8576421569620596, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012187744287075474, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00019089931447524577, 0.00018474484386388212, 0.00017900968668982387, 0.00017368212866131216, 0.0001685509632807225, 0.00016333791427314281, 0.00015942849859129637, 0.00015560787869617343, 0.00015185608936008066, 0.00014945000293664634, 0.00014607264893129468, 0.0001432896824553609, 0.00014164757158141583, 0.0001392114645568654, 0.00013621553080156446, 0.0001340551534667611, 0.00013190307072363794, 0.00012984307250007987, 0.0001278160052606836, 0.00012616341700777411, 0.0001243978040292859, 0.00012323912233114243, 0.00012188261462142691, 0.00012065970804542303, 0.00011920796532649547, 0.00011786595860030502, 0.00011683545744745061, 0.00011575424287002534, 0.00011453901242930442, 0.00011367436673026532, 0.00011304156214464456, 0.00011240885214647278 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 0.00014552358334185556, "first_to_last_ratio": 1.262638843036328, "imbalance_abs_log_ratio": 0.2332038508028458, "last_quartile_mean": 0.00011525352965691127, "mean": 0.00012953498708156985, "normalized": [ 1.1676304712442283, 1.1525836984023852, 1.1391521450612143, 1.1288261090295841, 1.1167600441451297, 1.1054423860149858, 1.094731137320412, 1.082319740349084, 1.067957453000188, 1.0563935463552665, 1.04560972624621, 1.0337959762176818, 1.0235535208863231, 1.0159040893770892, 1.0050628637064216, 0.9950792389307415, 0.9865814183778941, 0.9786230528286792, 0.9704418054120744, 0.9608381133357455, 0.9508407831226554, 0.9429749854457838, 0.9346839821902475, 0.9262276149685831, 0.9170542960948607, 0.9059488475652709, 0.8984648923618161, 0.8908037775288885, 0.8831483920181569, 0.8778579808382188, 0.8744051093280881, 0.8703028022960919 ], "peak_layer": 1, "peak_normalized": 1.1676304712442283, "population_cv": 0.0896642712448568, "rest_mean": 0.00013110964479252557, "spike_contrast": 0.9231344927669946, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012103183544240892, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00015124899800866842, 0.00014929991448298097, 0.00014756005839444697, 0.00014622247545048594, 0.00014465949789155275, 0.00014319346519187093, 0.00014180598373059183, 0.00014019827358424664, 0.00013833785487804562, 0.00013683992438018322, 0.0001354430423816666, 0.00013391274842433631, 0.0001325859921053052, 0.00013159512309357524, 0.00013019080506637692, 0.00012889757636003196, 0.00012779681128449738, 0.00012676592450588942, 0.00012570616672746837, 0.00012446215259842575, 0.00012316714855842292, 0.00012214825255796313, 0.00012107427755836397, 0.00011997888213954866, 0.00011879061639774591, 0.00011735207226593047, 0.0001163826382253319, 0.0001153902558144182, 0.00011439861555118114, 0.00011371332220733166, 0.00011326605454087257, 0.0001127346622524783 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00029742828155576717, "first_to_last_ratio": 2.2781890121807993, "imbalance_abs_log_ratio": 0.8233808344454435, "last_quartile_mean": 0.00013055469935352448, "mean": 0.00019329176166138495, "normalized": [ 1.8891850069388627, 1.7801603926271052, 1.6773544463608039, 1.566500262900107, 1.476456430986117, 1.3754925511314788, 1.3084504582544147, 1.236424592515421, 1.172965600373877, 1.150871569148191, 1.0802294364746856, 1.0319209662753672, 1.0080063247818516, 0.9698985570693867, 0.9067931019504507, 0.8756163462587222, 0.8444435806568995, 0.8113939671049883, 0.7809677247391495, 0.7613442367276676, 0.7402582678385915, 0.7293562882295931, 0.7162426695419845, 0.7062419232587185, 0.6940340549797547, 0.6866100033591334, 0.6831688143117768, 0.6753320518823956, 0.6715919825156708, 0.6686410475337363, 0.6640239865195754, 0.6600233567535221 ], "peak_layer": 1, "peak_normalized": 1.8891850069388627, "population_cv": 0.357915029863326, "rest_mean": 0.00020341356920888993, "spike_contrast": 0.6815376252628051, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001386340009048581, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00036516389809548855, 0.0003440903383307159, 0.0003242187958676368, 0.0003027915954589844, 0.00028538686456158757, 0.00026587137836031616, 0.0002529126941226423, 0.00023899068764876574, 0.00022672458726447076, 0.00022245399304665625, 0.0002087994507746771, 0.00019946182146668434, 0.00019483931828290224, 0.00018747340072877705, 0.00017527563613839447, 0.00016924942610785365, 0.00016322398732881993, 0.00015683576930314302, 0.0001509546273155138, 0.00014716156874783337, 0.0001430858246749267, 0.0001409785618307069, 0.00013844380737282336, 0.00013651074550580233, 0.00013415106514003128, 0.00013271605712361634, 0.00013205090363044292, 0.00013053612201474607, 0.00012981319741811603, 0.00012924280599690974, 0.00012835036613978446, 0.00012757707736454904 ] } } }, "pre_attention_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "de71d4cf2d509a1063f6eac7f50c4885e9a058dee7e4b993ad40f5527f528d00", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.02802751213312149, 0.01413465291261673, 0.014225807972252369, 0.01433455292135477, 0.01440478302538395, 0.009703479707241058, 0.00977881159633398, 0.009858808480203152, 0.009950309991836548, 0.00752287870272994, 0.007571155205368996, 0.007693085819482803, 0.007788818795233965, 0.006353048607707024, 0.006389216054230928, 0.0064707002602517605, 0.00656962301582098, 0.005553483963012695, 0.0056249531917274, 0.005688921548426151, 0.005768724251538515, 0.004966245498508215, 0.005057469941675663, 0.005098348017781973, 0.005216081161051989, 0.0046758754178881645, 0.004727107938379049, 0.004777040798217058, 0.004837613552808762, 0.0044186655431985855, 0.004510571248829365, 0.0045849340967834 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.0001580166108396952, "first_to_last_ratio": 1.9713740353643698, "imbalance_abs_log_ratio": 0.6787307795042399, "last_quartile_mean": 8.015557068574708e-05, "mean": 0.00011843875881822896, "normalized": [ 0.8918679597819287, 1.5814819593588525, 1.5722239987443252, 1.3246067564285042, 1.1252650527356378, 1.512743589128775, 1.3788112428745087, 1.2863035124436846, 1.097095055843533, 1.468154353173571, 1.1523254606068964, 1.0839888571909224, 1.054518217028451, 1.146335080549274, 1.1665208527784754, 0.8931136212023528, 0.8203804970547371, 1.073741223277434, 0.919654742394771, 0.8223971330190447, 0.7711288002244273, 0.8470775332584999, 0.8298711629738247, 0.7662487463744663, 0.7210382462310316, 0.7071376059209309, 0.7189559392079661, 0.727980518763353, 0.6111916361855597, 0.6986829946379066, 0.6244678178312321, 0.6046898327751223 ], "peak_layer": 2, "peak_normalized": 1.5814819593588525, "population_cv": 0.2904715201464359, "rest_mean": 0.00012310891098523926, "spike_contrast": 0.7572151875143335, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.321993711637333e-05, "top_five_layers": [ 2, 3, 6, 10, 7 ], "top_five_spike_overlap": 0 }, "values": [ 0.00010563173418631777, 0.0001873087603598833, 0.00018621225899551064, 0.00015688478015363216, 0.0001332749961875379, 0.00017916747310664505, 0.00016330469225067645, 0.00015234819147735834, 0.00012993857671972364, 0.00017388637934345752, 0.0001364799973089248, 0.0001283862948184833, 0.00012489582877606153, 0.00013577050413005054, 0.00013816128193866462, 0.00010577926877886057, 9.71648478298448e-05, 0.00012717257777694613, 0.00010892276623053476, 9.74036956904456e-05, 9.133153798757121e-05, 0.0001003268116619438, 9.828891052166e-05, 9.07535504666157e-05, 8.539887494407594e-05, 8.375250035896897e-05, 8.515224908478558e-05, 8.622110908618197e-05, 7.238877878990024e-05, 8.275114669231698e-05, 7.396119326585904e-05, 7.161871326388791e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.0001664285200604354, "first_to_last_ratio": 1.9719337364651075, "imbalance_abs_log_ratio": 0.6790146534176558, "last_quartile_mean": 8.439863722742302e-05, "mean": 0.0001245867656507471, "normalized": [ 0.8984334461398115, 1.5927952483854981, 1.5658595485525297, 1.3198735865176374, 1.131213894321221, 1.5125315092450449, 1.3796810696180641, 1.286366052942558, 1.0960606282761096, 1.4536884557404346, 1.1543342746982923, 1.0797559630746076, 1.0497530570909623, 1.1457182987306291, 1.1551962694199982, 0.9023721663305979, 0.8238710779144349, 1.0632503993742801, 0.9211624832492573, 0.8274335806257342, 0.7739032412019898, 0.850990856837025, 0.8281317613098801, 0.7681943949939335, 0.7208251570035091, 0.7089546837555952, 0.7203162530736348, 0.7236174125342815, 0.6145740290254166, 0.6994991973458914, 0.6274381906330746, 0.6042038120380651 ], "peak_layer": 2, "peak_normalized": 1.5927952483854981, "population_cv": 0.2889665774838781, "rest_mean": 0.00012946851159169995, "spike_contrast": 0.7586812913967177, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.822533756960183e-05, "top_five_layers": [ 2, 3, 6, 10, 7 ], "top_five_spike_overlap": 0 }, "values": [ 0.00011193291720701382, 0.0001984412083402276, 0.0001950853766174987, 0.000164438781212084, 0.00014093428035266697, 0.00018844140868168324, 0.00017189000209327787, 0.00016026418597903103, 0.00013655464863404632, 0.00018111034296453, 0.00014381477376446128, 0.0001345233031315729, 0.00013078533811494708, 0.00014274133718572557, 0.00014392216689884663, 0.00011242362961638719, 0.00010264343291055411, 0.0001324669283349067, 0.00011476465442683548, 0.00010308727360097691, 9.641810174798593e-05, 0.00010602219845168293, 0.00010317425767425448, 9.570685506332666e-05, 8.980527491075918e-05, 8.832637104205787e-05, 8.974187221610919e-05, 9.015315299620852e-05, 7.656779052922502e-05, 8.714834257261828e-05, 7.817049481673166e-05, 7.527579873567447e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.14759047608822584, "first_to_last_ratio": 1.9719337947467823, "imbalance_abs_log_ratio": 0.6790146829732503, "last_quartile_mean": 0.07484555337578058, "mean": 0.1104847895912826, "normalized": [ 0.8984334974418968, 1.5927952138130326, 1.565859646322909, 1.3198735575362357, 1.1312139236939245, 1.5125315007911495, 1.3796810264894532, 1.286366010451999, 1.0960606168617568, 1.4536884922319049, 1.1543342013148132, 1.0797558953237962, 1.0497530983161607, 1.1457182562234747, 1.1551962948455465, 0.9023721940039697, 0.8238711061937815, 1.0632504861804335, 0.9211625151546727, 0.8274335809310764, 0.7739032642161865, 0.8509909070635955, 0.8281317391228538, 0.7681943896830757, 0.7208251689669645, 0.7089546589522648, 0.7203162343708721, 0.7236173971219663, 0.6145740294939214, 0.6994991437375198, 0.6274381319743458, 0.6042038211744466 ], "peak_layer": 2, "peak_normalized": 1.5927952138130326, "population_cv": 0.28896658018674404, "rest_mean": 0.11481397047086998, "spike_contrast": 0.7586813040631772, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.08710721284151077, "top_five_layers": [ 2, 3, 6, 10, 7 ], "top_five_spike_overlap": 0 }, "values": [ 0.09926323592662811, 0.1759796440601349, 0.1730036735534668, 0.14582595229148865, 0.12498193234205246, 0.16711172461509705, 0.15243376791477203, 0.14212387800216675, 0.12109802663326263, 0.16061046719551086, 0.1275363713502884, 0.1192966029047966, 0.11598175019025803, 0.12658444046974182, 0.12763161957263947, 0.09969840198755264, 0.0910252258181572, 0.11747300624847412, 0.10177444666624069, 0.09141882508993149, 0.08550453931093216, 0.09402155131101608, 0.09149596095085144, 0.08487379550933838, 0.0796402171254158, 0.0783287063241005, 0.07958398759365082, 0.07994871586561203, 0.06790108233690262, 0.07728401571512222, 0.069322369992733, 0.06675533205270767 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 5.093040158499207e-05, "first_to_last_ratio": 0.8860093039525244, "imbalance_abs_log_ratio": 0.12102782735710801, "last_quartile_mean": 5.748291960117058e-05, "mean": 5.7093861016710434e-05, "normalized": [ 0.4337136137178416, 0.9185569667519601, 0.9444095926066538, 0.8728787927927439, 0.7849140301431445, 1.1296292591692207, 1.0726593944235638, 0.9796135302182263, 0.9386642177229508, 1.3046353648934683, 1.0461832118506482, 1.0032917361864164, 0.9662905330123907, 1.0760080966276753, 1.1828739682986207, 0.9087955578115051, 0.9194941496879511, 1.198292373503509, 1.0946132296674007, 1.0673695485587213, 0.9244260207272295, 1.0525679458300632, 1.125653431968099, 0.9999504891362015, 0.9629027708994021, 1.0063234987710272, 0.9975933861195996, 1.0791525148711678, 0.9412468232151083, 1.085173475571988, 1.0031556955576197, 0.9789667796878816 ], "peak_layer": 10, "peak_normalized": 1.3046353648934683, "population_cv": 0.1441687540611157, "rest_mean": 5.695535414647397e-05, "spike_contrast": 1.0155638391297448, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.784179811598733e-05, "top_five_layers": [ 10, 18, 15, 6, 23 ], "top_five_spike_overlap": 1 }, "values": [ 2.4762384782661684e-05, 5.2443963795667514e-05, 5.3919990023132414e-05, 4.9836020480142906e-05, 4.481377254705876e-05, 6.449489592341706e-05, 6.124226638348773e-05, 5.592991874436848e-05, 5.3591964388033375e-05, 7.448667020071298e-05, 5.973063889541663e-05, 5.728179894504137e-05, 5.516925739357248e-05, 6.143345672171563e-05, 6.75348419463262e-05, 5.1886647270293906e-05, 5.249747118796222e-05, 6.84151382301934e-05, 6.249569560168311e-05, 6.094024865888059e-05, 5.277905074763112e-05, 6.009516800986603e-05, 6.426790059776977e-05, 5.7091034250333905e-05, 5.4975836974335834e-05, 5.74548939766828e-05, 5.695645813830197e-05, 6.161298369988799e-05, 5.373941530706361e-05, 6.19567435933277e-05, 5.727403186028823e-05, 5.58929932594765e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 5.801741053801379e-05, "first_to_last_ratio": 0.954195857595535, "imbalance_abs_log_ratio": 0.046886327142486896, "last_quartile_mean": 6.08024129178375e-05, "mean": 6.296396202287724e-05, "normalized": [ 0.4796364558374551, 0.9568353419130547, 0.9804638447804884, 0.8949545354119591, 0.8237377234766136, 1.1597846916175047, 1.085649359633547, 0.9904450076993521, 0.9595078978373608, 1.2941748206155441, 1.0680753760655664, 1.0231559828609242, 0.9904019047543832, 1.096490963646882, 1.1850351594596897, 0.9422215946487361, 0.9418631353047864, 1.1705024179981143, 1.0926353878857977, 1.0382275689133567, 0.9320833313178482, 1.0784583323883348, 1.0985080196909318, 0.9917906747078287, 0.9476388143731139, 0.9855066582268529, 0.978396290111904, 1.028407839812513, 0.8982055143685478, 1.0282804954549911, 0.9451124964564701, 0.9138123627295472 ], "peak_layer": 10, "peak_normalized": 1.2941748206155441, "population_cv": 0.13431233496292436, "rest_mean": 6.285090866084729e-05, "spike_contrast": 1.0115120295379672, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.357445017783903e-05, "top_five_layers": [ 10, 15, 18, 6, 23 ], "top_five_spike_overlap": 1 }, "values": [ 3.019981159013696e-05, 6.0246144130360335e-05, 6.173388828756288e-05, 5.634988337988034e-05, 5.186579073779285e-05, 7.302463927771896e-05, 6.835678505012766e-05, 6.236234185053036e-05, 6.041441884008236e-05, 8.148637425620109e-05, 6.725025741616264e-05, 6.442195444833487e-05, 6.235962791834027e-05, 6.903941539349034e-05, 7.461450877599418e-05, 5.9326004702597857e-05, 5.930343468207866e-05, 7.369946979451925e-05, 6.879665306769311e-05, 6.537092122016475e-05, 5.8687659475253895e-05, 6.790400948375463e-05, 6.916641723364592e-05, 6.244707037694752e-05, 5.966709431959316e-05, 6.205140380188823e-05, 6.160370685392991e-05, 6.475263216998428e-05, 5.655457789544016e-05, 6.474461406469345e-05, 5.950802733423188e-05, 5.753724690293893e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 1.7888660295284353e-05, "first_to_last_ratio": 0.41855462748958633, "imbalance_abs_log_ratio": 0.8709478659353102, "last_quartile_mean": 4.2739129185065394e-05, "mean": 3.1986895351110434e-05, "normalized": [ 0.2593955501951102, 0.4997829486650695, 0.48480994473028793, 0.4968760363401999, 0.5474125689219601, 0.7788406177150305, 0.7554473604850604, 0.6514322393717875, 0.7578135841604356, 0.9674810188795085, 0.9032772556875033, 0.9019220651946442, 0.8311356963419073, 0.9587970177975769, 1.1401394869948454, 0.8953886492354074, 1.0262272842074007, 1.1413916918814533, 1.174037958462901, 1.194927715786966, 1.0267140632187253, 1.3434898267078448, 1.3729067467361273, 1.2011929483563188, 1.1735959904166378, 1.3266745689353452, 1.2739952988364465, 1.4049535244931144, 1.2752889026856997, 1.4680979746684197, 1.4182292846928506, 1.3483241791974143 ], "peak_layer": 30, "peak_normalized": 1.4680979746684197, "population_cv": 0.3184955307507873, "rest_mean": 3.066251991657299e-05, "spike_contrast": 1.276428773763581, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.913852269761264e-05, "top_five_layers": [ 30, 31, 28, 23, 32 ], "top_five_spike_overlap": 1 }, "values": [ 8.297258318634704e-06, 1.5986504877218977e-05, 1.5507564967265353e-05, 1.589352177688852e-05, 1.7510028555989265e-05, 2.491269333404489e-05, 2.416441566310823e-05, 2.083729486912489e-05, 2.4240103812189773e-05, 3.094671410508454e-05, 2.889303505071439e-05, 2.8849686714238487e-05, 2.6585450541460887e-05, 3.066893987124786e-05, 3.6469522456172854e-05, 2.8640703021665104e-05, 3.282582474639639e-05, 3.6509576602838933e-05, 3.755382931558415e-05, 3.822202779701911e-05, 3.2841395295690745e-05, 4.2974068492185324e-05, 4.391502443468198e-05, 3.842243313556537e-05, 3.7539692129939795e-05, 4.243620060151443e-05, 4.0751154301688075e-05, 4.494010136113502e-05, 4.0792532672639936e-05, 4.695989628089592e-05, 4.5364751713350415e-05, 4.3128704419359565e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00022091567916504573, "first_to_last_ratio": 1.232986382535498, "imbalance_abs_log_ratio": 0.20943917994887448, "last_quartile_mean": 0.00017917122386279516, "mean": 0.0002198539991695725, "normalized": [ 0.4672441326905094, 1.0299890566632133, 1.08986441571188, 1.0084939820045402, 0.8814237539069011, 1.2693826047962031, 1.1886514905905403, 1.1035827467935222, 1.0473307057033197, 1.4274044616309502, 1.1442998336482333, 1.0911283607583675, 1.1094038041384326, 1.1946873974128753, 1.2511466100570947, 0.9923165311956169, 0.9239703707694655, 1.2473578196113966, 1.1181336030529359, 1.0160745414007064, 0.9397394354798955, 0.9905446518023159, 1.0121913654482901, 0.9359943959598221, 0.8839446941554127, 0.8510229499852975, 0.8648513533654478, 0.8786898836618169, 0.7455559493639876, 0.8399702482950041, 0.7392957942269532, 0.7163130557190521 ], "peak_layer": 10, "peak_normalized": 1.4274044616309502, "population_cv": 0.18952709251819305, "rest_mean": 0.0002217885959436427, "spike_contrast": 0.9441746799407325, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00020940717658959329, "top_five_layers": [ 10, 6, 15, 18, 14 ], "top_five_spike_overlap": 0 }, "values": [ 0.00010272549116052687, 0.00022644721320830286, 0.00023961105034686625, 0.00022172143508214504, 0.0001937845372594893, 0.0002790788421407342, 0.0002613297838252038, 0.00024262708029709756, 0.00023025984410196543, 0.000313820579322055, 0.00025157889467664063, 0.00023988893372006714, 0.00024390686303377151, 0.000262656802078709, 0.00027506958576850593, 0.0002181647578254342, 0.00020313858112785965, 0.00027423660503700376, 0.0002458261442370713, 0.00022338805138133466, 0.00020660547306761146, 0.00021777520305477083, 0.00022253431961871684, 0.00020578211115207523, 0.0001943387760547921, 0.00018710079893935472, 0.0001901410287246108, 0.00019318348495289683, 0.00016391345707233995, 0.00018467081827111542, 0.00016253713693004102, 0.00015748428995721042 ] } } }, "pre_mlp_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "2999a257265024350fd2665e2a8d452a03cd662193b4245d15290408ad29922d", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.014061377383768559, 0.014165290631353855, 0.014281988143920898, 0.014350161887705326, 0.00966146495193243, 0.009741888381540775, 0.009820535778999329, 0.00991273857653141, 0.0074927168898284435, 0.007546746172010899, 0.00766373984515667, 0.007762390188872814, 0.006325984839349985, 0.006375366821885109, 0.006445344537496567, 0.006555322092026472, 0.005531130358576775, 0.005611324217170477, 0.0056777093559503555, 0.00574630219489336, 0.004946626722812653, 0.0050392793491482735, 0.005084683652967215, 0.005203457549214363, 0.004667261149734259, 0.004717131145298481, 0.004768412560224533, 0.004826710093766451, 0.0044052014127373695, 0.004504534415900707, 0.004574235528707504, 0.004644096828997135 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 3.729895206561196e-05, "first_to_last_ratio": 1.2469721805352876, "imbalance_abs_log_ratio": 0.22071835733540474, "last_quartile_mean": 2.9911615229138988e-05, "mean": 3.340667609563752e-05, "normalized": [ 1.308752778077389, 1.2456464596610868, 1.0236956931411572, 0.9922285647386262, 1.26903181062375, 1.1170846373362797, 1.0287316533628064, 0.9469237624012169, 1.1300261751959062, 1.1132698789554223, 1.0126240729566007, 0.9379202574439188, 1.1130503370514597, 1.070713488608195, 0.9897642722064988, 0.922085688720865, 1.036831181462568, 0.9898105546019274, 0.9395130252451869, 0.9001967292772246, 0.9975952535130206, 0.9404117749145336, 0.9224862220158919, 0.8885787590271026, 0.9461670180110204, 0.8974372095121397, 0.8904691498559152, 0.8684437934704528, 0.9244147277585869, 0.9015689206268677, 0.8740901912628669, 0.8604359589635168 ], "peak_layer": 1, "peak_normalized": 1.308752778077389, "population_cv": 0.1164905978879517, "rest_mean": 3.378375206213807e-05, "spike_contrast": 0.9285666618330386, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.137046587653458e-05, "top_five_layers": [ 1, 5, 2, 9, 6 ], "top_five_spike_overlap": 0 }, "values": [ 4.3721080146497115e-05, 4.161290780757554e-05, 3.419827044126578e-05, 3.314705827506259e-05, 4.239413465256803e-05, 3.73180846509058e-05, 3.4366505133220926e-05, 3.1633575417799875e-05, 3.7750418414361775e-05, 3.719064625329338e-05, 3.3828404411906376e-05, 3.1332798243965954e-05, 3.718331208801828e-05, 3.5768978705164045e-05, 3.306473445263691e-05, 3.080381793552078e-05, 3.463708344497718e-05, 3.3066280593629926e-05, 3.138600732199848e-05, 3.007258055731654e-05, 3.332634150865488e-05, 3.1416031561093405e-05, 3.0817198421573266e-05, 2.9684462788281962e-05, 3.1608295103069395e-05, 2.9980394174344838e-05, 2.9747614462394267e-05, 2.9011820515734144e-05, 3.0881623388268054e-05, 3.01184209092753e-05, 2.9200447897892445e-05, 2.874430538213346e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 6.705871965095866e-05, "first_to_last_ratio": 0.7180297289018083, "imbalance_abs_log_ratio": 0.33124430563536084, "last_quartile_mean": 9.33926785364747e-05, "mean": 8.228966771639534e-05, "normalized": [ 0.8685826408392191, 0.8249006353131848, 0.7154905339803773, 0.6884730939760333, 0.9470487095509537, 0.8638069620260335, 0.8256155016985178, 0.7853664463613442, 0.9851686390305121, 0.9692511243547932, 0.9083804947955202, 0.8729855457896263, 1.0422548534804916, 1.0255492616133721, 0.9871528463964067, 0.9398223255303129, 1.0958375595009802, 1.0726325633807163, 1.0495259774381736, 1.0230838788771643, 1.1576441025653552, 1.1142532603054807, 1.1008567438213033, 1.056908774400763, 1.159063578749664, 1.1192434432436902, 1.1032241584740294, 1.1060435702772593, 1.1786235078649367, 1.1628365878275773, 1.141840825041133, 1.1085318534950752 ], "peak_layer": 29, "peak_normalized": 1.1786235078649367, "population_cv": 0.1341844256762087, "rest_mean": 8.049536754023629e-05, "spike_contrast": 1.1426606459269575, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.197888866765424e-05, "top_five_layers": [ 29, 30, 25, 21, 31 ], "top_five_spike_overlap": 2 }, "values": [ 7.14753768988885e-05, 6.788079917896539e-05, 5.887747829547152e-05, 5.66542221349664e-05, 7.793232362018898e-05, 7.108238787623122e-05, 6.793962529627606e-05, 6.462754390668124e-05, 8.106919995043427e-05, 7.975935295689851e-05, 7.475032907677814e-05, 7.183769048424438e-05, 8.576680556870997e-05, 8.4392107964959e-05, 8.123247971525416e-05, 7.733766688033938e-05, 9.017610864248127e-05, 8.826657722238451e-05, 8.636514394311234e-05, 8.418923243880272e-05, 9.526214853394777e-05, 9.169153054244816e-05, 9.0589135652408e-05, 8.697267185198143e-05, 9.537895675748587e-05, 9.210217103827745e-05, 9.078394941752777e-05, 9.101595787797123e-05, 9.698853682493791e-05, 9.56894364207983e-05, 9.396170207764953e-05, 9.122071787714958e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.05946834245696664, "first_to_last_ratio": 0.7180297222365345, "imbalance_abs_log_ratio": 0.33124431491808765, "last_quartile_mean": 0.08282156102359295, "mean": 0.07297530013602227, "normalized": [ 0.8685826122359729, 0.8249005917343738, 0.71549052911611, 0.6884731350635278, 0.9470486699630107, 0.8638070113577097, 0.8256154769796886, 0.7853664269052657, 0.9851686341386658, 0.9692511556795396, 0.9083805454214465, 0.8729855579949275, 1.0422548040385637, 1.0255492365750694, 0.9871528950273681, 0.9398223256645052, 1.0958375053304956, 1.07263252629438, 1.0495259690486791, 1.0230837911446407, 1.1576441435704576, 1.1142533045483842, 1.1008568124470575, 1.0569088284972943, 1.1590636022344982, 1.119243411839159, 1.1032241423789504, 1.1060435591240758, 1.1786235037173105, 1.1628366076958818, 1.141840809884862, 1.1085318743481278 ], "peak_layer": 29, "peak_normalized": 1.1786235037173105, "population_cv": 0.13418442668798974, "rest_mean": 0.07138409617322462, "spike_contrast": 1.1426607032635474, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.08156780153512955, "top_five_layers": [ 29, 30, 25, 21, 31 ], "top_five_spike_overlap": 2 }, "values": [ 0.06338507682085037, 0.0601973682641983, 0.05221313610672951, 0.050241533666849136, 0.0691111609339714, 0.06303657591342926, 0.060249537229537964, 0.05731235072016716, 0.07189297676086426, 0.07073139399290085, 0.06628934293985367, 0.06370638310909271, 0.07605885714292526, 0.0748397633433342, 0.07203777879476547, 0.06858381628990173, 0.07996907085180283, 0.07827568054199219, 0.07658947259187698, 0.07465984672307968, 0.0844794288277626, 0.08131296932697296, 0.0803353562951088, 0.07712823897600174, 0.08458301424980164, 0.08167712390422821, 0.08050811290740967, 0.08071386069059372, 0.0860104039311409, 0.08485835045576096, 0.08332617580890656, 0.08089544624090195 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 7.408341275549901e-06, "first_to_last_ratio": 0.49124437919492936, "imbalance_abs_log_ratio": 0.7108135577065074, "last_quartile_mean": 1.5080765479069669e-05, "mean": 1.1844782648040564e-05, "normalized": [ 0.5657990507718705, 0.5382233871697658, 0.49942145823009426, 0.48906984415931193, 0.8062921756393574, 0.7122953827323945, 0.7013508394120195, 0.6911626993599347, 0.8940675881372303, 0.868722127857376, 0.9735911331661544, 0.854855860776803, 0.989483587379368, 1.0059088486455374, 1.0303729006301217, 0.9941025538400318, 1.1620912657632405, 1.127012997009919, 1.113401270263767, 1.066097770938846, 1.1826034542812638, 1.1643547935214302, 1.2049778184223279, 1.1791491538213539, 1.2365172470752577, 1.219921447624247, 1.2337761202582145, 1.2288278254433027, 1.3039217543779833, 1.3631114810840155, 1.2856635723507386, 1.3138525898567213 ], "peak_layer": 30, "peak_normalized": 1.3631114810840155, "population_cv": 0.2549101540724733, "rest_mean": 1.1420299651271543e-05, "spike_contrast": 1.237882653019464, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.4136990830593277e-05, "top_five_layers": [ 30, 32, 29, 31, 25 ], "top_five_spike_overlap": 1 }, "values": [ 6.701766778860474e-06, 6.3751390371180605e-06, 5.915538622502936e-06, 5.7929260037781205e-06, 9.550355571263935e-06, 8.436983989668079e-06, 8.307348252856173e-06, 8.186671948351432e-06, 1.0590036254143342e-05, 1.0289824786013924e-05, 1.1531975360412616e-05, 1.0125581866304856e-05, 1.1720218026312068e-05, 1.1914771675947122e-05, 1.220454305439489e-05, 1.1774928680097219e-05, 1.3764718460151926e-05, 1.334922399109928e-05, 1.3187996046326589e-05, 1.2627696378331166e-05, 1.4007680874783546e-05, 1.379152945446549e-05, 1.4272700354922563e-05, 1.3966765436634887e-05, 1.4646278032159898e-05, 1.4449704394792207e-05, 1.4613809980801307e-05, 1.4555198504240252e-05, 1.5444669770658948e-05, 1.614575921848882e-05, 1.5228405572997872e-05, 1.5562298358418047e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 5.6029601637419546e-05, "first_to_last_ratio": 0.6070213107493427, "imbalance_abs_log_ratio": 0.49919138022039955, "last_quartile_mean": 9.230252817360451e-05, "mean": 7.840498903988191e-05, "normalized": [ 0.6598133090619259, 0.6419677920829977, 0.6005256373772865, 0.5913463573822231, 0.8562376513989928, 0.8062968559250673, 0.7923156566726156, 0.768439227921366, 0.9790122729268586, 0.9665730330981341, 0.9163710065350301, 0.8881686280668529, 1.0631190324760253, 1.0511030519715439, 1.0188080221730096, 0.9712812246823179, 1.1361272291219966, 1.1138561419734825, 1.0918134348286874, 1.0639217496772315, 1.203414696914147, 1.1592426077267617, 1.1442664103435203, 1.097952968622282, 1.2042281788778202, 1.1625445130141736, 1.1452986211445626, 1.1473932954411665, 1.2233942816523278, 1.205318760675343, 1.1829469858778916, 1.146901364356358 ], "peak_layer": 29, "peak_normalized": 1.2233942816523278, "population_cv": 0.1898597249088972, "rest_mean": 7.605543874820728e-05, "spike_contrast": 1.1977126437531995, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.109256061492487e-05, "top_five_layers": [ 29, 30, 25, 21, 31 ], "top_five_spike_overlap": 2 }, "values": [ 5.173265526536852e-05, 5.033347770222463e-05, 4.708420601673424e-05, 4.636450466932729e-05, 6.713330367347226e-05, 6.321769615169615e-05, 6.212150037754327e-05, 6.024946924299002e-05, 7.675944652874023e-05, 7.578414806630462e-05, 7.184805872384459e-05, 6.963685154914856e-05, 8.335383608937263e-05, 8.241172326961532e-05, 7.987963181221858e-05, 7.615329377586022e-05, 8.907804294722155e-05, 8.733187860343605e-05, 8.560362039133906e-05, 8.341677312273532e-05, 9.435371612198651e-05, 9.089040395338088e-05, 8.971619536168873e-05, 8.608499047113582e-05, 9.441749716643244e-05, 9.114928980125114e-05, 8.97971258382313e-05, 8.996135875349864e-05, 9.592021524440497e-05, 9.450300422031432e-05, 9.274894546251744e-05, 8.992278890218586e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 4.8511469685763586e-05, "first_to_last_ratio": 0.5271352332510719, "imbalance_abs_log_ratio": 0.6402981537748061, "last_quartile_mean": 9.202850924339145e-05, "mean": 7.532146378252946e-05, "normalized": [ 0.548104082267993, 0.5416830199005969, 0.528884702689083, 0.5247448675895723, 0.7777813723908718, 0.7487613756360456, 0.7496059866168319, 0.7329069161583202, 0.9480676391902623, 0.9399123881532756, 0.9000936195631334, 0.883441930780847, 1.0560589316963704, 1.0534552095740957, 1.032095781768693, 0.9835620691098637, 1.1601754560133901, 1.1467788545782247, 1.1233340534265408, 1.0967375255670042, 1.2396083970871608, 1.1951467072205837, 1.1828066060767042, 1.1317734013234224, 1.244698956870482, 1.2025862582796845, 1.1881970083126314, 1.1882256015362047, 1.2689460104605979, 1.2589162611011824, 1.2307315535498626, 1.1921774555104672 ], "peak_layer": 29, "peak_normalized": 1.2689460104605979, "population_cv": 0.23486334953575583, "rest_mean": 7.254842300123225e-05, "spike_contrast": 1.244629176846487, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.029588400153444e-05, "top_five_layers": [ 29, 30, 25, 21, 31 ], "top_five_spike_overlap": 2 }, "values": [ 4.1284001781605184e-05, 4.080035796505399e-05, 3.9836369978729635e-05, 3.952455153921619e-05, 5.858363147126511e-05, 5.639780283672735e-05, 5.6461420172126964e-05, 5.520362174138427e-05, 7.140984234865755e-05, 7.079557690303773e-05, 6.77963689668104e-05, 6.654213939327747e-05, 7.954390457598493e-05, 7.934778841445222e-05, 7.773896504659206e-05, 7.408333476632833e-05, 8.738611359149218e-05, 8.637706196168438e-05, 8.461116522084922e-05, 8.26078758109361e-05, 9.336911898571998e-05, 9.002019942272455e-05, 8.909072494134307e-05, 8.524682925781235e-05, 9.375254740007222e-05, 9.058055729838088e-05, 8.949673792812973e-05, 8.949889161158353e-05, 9.557887096889317e-05, 9.482341556577012e-05, 9.270050213672221e-05, 8.979655103757977e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 9.61669284151867e-05, "first_to_last_ratio": 0.8297833624837898, "imbalance_abs_log_ratio": 0.18659062131596024, "last_quartile_mean": 0.00011589401856326731, "mean": 0.00010716335873439675, "normalized": [ 0.9380742209455786, 0.8898820858744699, 0.7837417114644826, 0.7583102664636657, 1.0822070619439073, 0.9520749081713696, 0.9107982467806989, 0.8640017807748708, 1.0669538303069324, 1.0266950649381863, 0.9594029855154438, 0.8901506143407661, 1.0795289063552718, 1.0379165674077444, 0.9706971366471856, 0.914979822300863, 1.052939971267624, 1.0225707935964177, 0.9851238030379684, 0.9636652893153618, 1.0990044510799908, 1.0523904893769196, 1.0422234788441622, 1.0049019599938585, 1.0962247910521359, 1.0599378706264646, 1.05039930496957, 1.0583562345651827, 1.1199788667078585, 1.1100529546792268, 1.0850973923636869, 1.0717171382921347 ], "peak_layer": 29, "peak_normalized": 1.1199788667078585, "population_cv": 0.09156717959150794, "rest_mean": 0.00010599351123716958, "spike_contrast": 1.0706366257223145, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00011348053521942347, "top_five_layers": [ 29, 30, 21, 25, 31 ], "top_five_spike_overlap": 2 }, "values": [ 0.00010052718425868079, 9.536275319987908e-05, 8.398839418077841e-05, 8.12630751170218e-05, 0.00011597294360399246, 0.00010202754492638633, 9.760419925441965e-05, 9.25893327803351e-05, 0.00011433835607022047, 0.00011002409155480564, 0.00010281284630764276, 9.539152961224318e-05, 0.000115685943455901, 0.0001112266254494898, 0.0001040231654769741, 9.805231093196198e-05, 0.00011283658386673778, 0.00010958212078548968, 0.00010556917550275102, 0.00010326960909878835, 0.00011777300824178383, 0.0001127776995417662, 0.00011168816854478791, 0.00010768866923172027, 0.00011747513053705916, 0.00011358650226611644, 0.00011256431753281504, 0.00011341700883349404, 0.00012002069706795737, 0.00011895700299646705, 0.00011628268111962825, 0.00011484880815260112 ] } } } }, "selector": { "expected_uniform_count": 8, "identities_unique": true, "identity_order_sha256": "97fd9721a263024f6a746638f32e0ad9a988cd4c111c00e9e178c1022e5c9d5e", "passed": true, "uniform_count": 8, "uniform_indices": [ 48, 49, 50, 51, 52, 53, 54, 55 ], "visit_count": 65, "visits": [ { "branch": "attention", "coefficient": "detached_learned", "group": 1, "index": 0, "kind": "depth", "layer": 1, "offset": 1, "sources": 1, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 1, "index": 1, "kind": "depth", "layer": 1, "offset": 1, "sources": 2, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 1, "index": 2, "kind": "depth", "layer": 2, "offset": 2, "sources": 2, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 1, "index": 3, "kind": "depth", "layer": 2, "offset": 2, "sources": 2, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 1, "index": 4, "kind": "depth", "layer": 3, "offset": 3, "sources": 2, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 1, "index": 5, "kind": "depth", "layer": 3, "offset": 3, "sources": 2, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 1, "index": 6, "kind": "depth", "layer": 4, "offset": 4, "sources": 2, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 1, "index": 7, "kind": "depth", "layer": 4, "offset": 4, "sources": 2, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 2, "index": 8, "kind": "depth", "layer": 5, "offset": 1, "sources": 2, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 2, "index": 9, "kind": "depth", "layer": 5, "offset": 1, "sources": 3, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 2, "index": 10, "kind": "depth", "layer": 6, "offset": 2, "sources": 3, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 2, "index": 11, "kind": "depth", "layer": 6, "offset": 2, "sources": 3, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 2, "index": 12, "kind": "depth", "layer": 7, "offset": 3, "sources": 3, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 2, "index": 13, "kind": "depth", "layer": 7, "offset": 3, "sources": 3, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 2, "index": 14, "kind": "depth", "layer": 8, "offset": 4, "sources": 3, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 2, "index": 15, "kind": "depth", "layer": 8, "offset": 4, "sources": 3, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 3, "index": 16, "kind": "depth", "layer": 9, "offset": 1, "sources": 3, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 3, "index": 17, "kind": "depth", "layer": 9, "offset": 1, "sources": 4, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 3, "index": 18, "kind": "depth", "layer": 10, "offset": 2, "sources": 4, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 3, "index": 19, "kind": "depth", "layer": 10, "offset": 2, "sources": 4, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 3, "index": 20, "kind": "depth", "layer": 11, "offset": 3, "sources": 4, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 3, "index": 21, "kind": "depth", "layer": 11, "offset": 3, "sources": 4, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 3, "index": 22, "kind": "depth", "layer": 12, "offset": 4, "sources": 4, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 3, "index": 23, "kind": "depth", "layer": 12, "offset": 4, "sources": 4, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 4, "index": 24, "kind": "depth", "layer": 13, "offset": 1, "sources": 4, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 4, "index": 25, "kind": "depth", "layer": 13, "offset": 1, "sources": 5, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 4, "index": 26, "kind": "depth", "layer": 14, "offset": 2, "sources": 5, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 4, "index": 27, "kind": "depth", "layer": 14, "offset": 2, "sources": 5, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 4, "index": 28, "kind": "depth", "layer": 15, "offset": 3, "sources": 5, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 4, "index": 29, "kind": "depth", "layer": 15, "offset": 3, "sources": 5, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 4, "index": 30, "kind": "depth", "layer": 16, "offset": 4, "sources": 5, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 4, "index": 31, "kind": "depth", "layer": 16, "offset": 4, "sources": 5, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 5, "index": 32, "kind": "depth", "layer": 17, "offset": 1, "sources": 5, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 5, "index": 33, "kind": "depth", "layer": 17, "offset": 1, "sources": 6, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 5, "index": 34, "kind": "depth", "layer": 18, "offset": 2, "sources": 6, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 5, "index": 35, "kind": "depth", "layer": 18, "offset": 2, "sources": 6, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 5, "index": 36, "kind": "depth", "layer": 19, "offset": 3, "sources": 6, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 5, "index": 37, "kind": "depth", "layer": 19, "offset": 3, "sources": 6, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 5, "index": 38, "kind": "depth", "layer": 20, "offset": 4, "sources": 6, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 5, "index": 39, "kind": "depth", "layer": 20, "offset": 4, "sources": 6, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 6, "index": 40, "kind": "depth", "layer": 21, "offset": 1, "sources": 6, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 6, "index": 41, "kind": "depth", "layer": 21, "offset": 1, "sources": 7, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 6, "index": 42, "kind": "depth", "layer": 22, "offset": 2, "sources": 7, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 6, "index": 43, "kind": "depth", "layer": 22, "offset": 2, "sources": 7, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 6, "index": 44, "kind": "depth", "layer": 23, "offset": 3, "sources": 7, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 6, "index": 45, "kind": "depth", "layer": 23, "offset": 3, "sources": 7, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 6, "index": 46, "kind": "depth", "layer": 24, "offset": 4, "sources": 7, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 6, "index": 47, "kind": "depth", "layer": 24, "offset": 4, "sources": 7, "use_uniform": false }, { "branch": "attention", "coefficient": "uniform", "group": 7, "index": 48, "kind": "depth", "layer": 25, "offset": 1, "sources": 7, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 7, "index": 49, "kind": "depth", "layer": 25, "offset": 1, "sources": 8, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 7, "index": 50, "kind": "depth", "layer": 26, "offset": 2, "sources": 8, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 7, "index": 51, "kind": "depth", "layer": 26, "offset": 2, "sources": 8, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 7, "index": 52, "kind": "depth", "layer": 27, "offset": 3, "sources": 8, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 7, "index": 53, "kind": "depth", "layer": 27, "offset": 3, "sources": 8, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 7, "index": 54, "kind": "depth", "layer": 28, "offset": 4, "sources": 8, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 7, "index": 55, "kind": "depth", "layer": 28, "offset": 4, "sources": 8, "use_uniform": true }, { "branch": "attention", "coefficient": "detached_learned", "group": 8, "index": 56, "kind": "depth", "layer": 29, "offset": 1, "sources": 8, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 8, "index": 57, "kind": "depth", "layer": 29, "offset": 1, "sources": 9, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 8, "index": 58, "kind": "depth", "layer": 30, "offset": 2, "sources": 9, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 8, "index": 59, "kind": "depth", "layer": 30, "offset": 2, "sources": 9, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 8, "index": 60, "kind": "depth", "layer": 31, "offset": 3, "sources": 9, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 8, "index": 61, "kind": "depth", "layer": 31, "offset": 3, "sources": 9, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 8, "index": 62, "kind": "depth", "layer": 32, "offset": 4, "sources": 9, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 8, "index": 63, "kind": "depth", "layer": 32, "offset": 4, "sources": 9, "use_uniform": false }, { "branch": "output", "coefficient": "detached_learned", "group": null, "index": 64, "kind": "output", "layer": null, "offset": null, "sources": 9, "use_uniform": false } ], "visits_sha256": "c9df79ea46ffe906b779be180e1a1ca10bc72a061b713a1b39e60845020b9b00" } }, "uniform_groups_6_7_only": { "bits_per_byte": 7.959341947457508, "forward": { "activation_sha256": { "attention_branch_output": "676759a2d7c0615ad7e1f246e90927fb829d00c8b6c3745032ae6b27ab7d902a", "mlp_branch_output": "afe29f3ef4d2992353f0bb59fe7c5c2c5050fca671a53184825a8d63cd83a2bc", "post_attention_state": "31853ffb07b24cc59e774bb20656840b4358b3080b42516d380f2dc0f46404c8", "post_mlp_state": "7fd9c911e93175896ad618842c120e77b3f017c17b7eec9c2bfb01fc6dbc8ac2", "pre_attention_input": "de71d4cf2d509a1063f6eac7f50c4885e9a058dee7e4b993ad40f5527f528d00", "pre_mlp_input": "2999a257265024350fd2665e2a8d452a03cd662193b4245d15290408ad29922d" }, "logits_sha256": "873a309a197bfd08fa523edcb440d2697903a34abc02dad1c7f5991509c3c151", "loss_nats": 5.516995429992676, "mixer_summary_sha256": "8450b24bdc777885a3357dc7b8080e7410acbe96943686433da76f379295133a" }, "loss_nats": 5.516995429992676, "loss_scale": 1.0, "mixers": [ { "branch": "attention", "entropy_mean": -0.0, "group": 1, "latest_source_mass": 1.0, "layer": 1, "max_source_mass": 1.0, "mixer_index": 0, "normalized_entropy": 1.0, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 1.0, "median": 1.0, "p05": 1.0, "p95": 1.0 } ], "sources": 1 }, { "branch": "mlp", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 1, "max_source_mass": 0.5, "mixer_index": 1, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 2, "max_source_mass": 0.5, "mixer_index": 2, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 2, "max_source_mass": 0.5, "mixer_index": 3, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 3, "max_source_mass": 0.5, "mixer_index": 4, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 3, "max_source_mass": 0.5, "mixer_index": 5, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 4, "max_source_mass": 0.5, "mixer_index": 6, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 4, "max_source_mass": 0.5, "mixer_index": 7, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6931471824645996, "group": 2, "latest_source_mass": 0.5, "layer": 5, "max_source_mass": 0.5, "mixer_index": 8, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "completed_group_1", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 5, "max_source_mass": 0.3333333432674408, "mixer_index": 9, "normalized_entropy": 1.0000000180537645, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 6, "max_source_mass": 0.3333333432674408, "mixer_index": 10, "normalized_entropy": 1.0000000180537645, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 6, "max_source_mass": 0.3333333432674408, "mixer_index": 11, "normalized_entropy": 1.0000000180537645, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 7, "max_source_mass": 0.3333333432674408, "mixer_index": 12, "normalized_entropy": 1.0000000180537645, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 7, "max_source_mass": 0.3333333432674408, "mixer_index": 13, "normalized_entropy": 1.0000000180537645, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 8, "max_source_mass": 0.3333333432674408, "mixer_index": 14, "normalized_entropy": 1.0000000180537645, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 8, "max_source_mass": 0.3333333432674408, "mixer_index": 15, "normalized_entropy": 1.0000000180537645, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0986123085021973, "group": 3, "latest_source_mass": 0.3333333432674408, "layer": 9, "max_source_mass": 0.3333333432674408, "mixer_index": 16, "normalized_entropy": 1.0000000180537645, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_2", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 9, "max_source_mass": 0.25, "mixer_index": 17, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 10, "max_source_mass": 0.25, "mixer_index": 18, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 10, "max_source_mass": 0.25, "mixer_index": 19, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 11, "max_source_mass": 0.25, "mixer_index": 20, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 11, "max_source_mass": 0.25, "mixer_index": 21, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 12, "max_source_mass": 0.25, "mixer_index": 22, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 12, "max_source_mass": 0.25, "mixer_index": 23, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3862943649291992, "group": 4, "latest_source_mass": 0.25, "layer": 13, "max_source_mass": 0.25, "mixer_index": 24, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_3", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 13, "max_source_mass": 0.20000000298023224, "mixer_index": 25, "normalized_entropy": 1.0000000186840277, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 14, "max_source_mass": 0.20000000298023224, "mixer_index": 26, "normalized_entropy": 1.0000000186840277, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 14, "max_source_mass": 0.20000000298023224, "mixer_index": 27, "normalized_entropy": 1.0000000186840277, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 15, "max_source_mass": 0.20000000298023224, "mixer_index": 28, "normalized_entropy": 1.0000000186840277, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 15, "max_source_mass": 0.20000000298023224, "mixer_index": 29, "normalized_entropy": 1.0000000186840277, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 16, "max_source_mass": 0.20000000298023224, "mixer_index": 30, "normalized_entropy": 1.0000000186840277, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 16, "max_source_mass": 0.20000000298023224, "mixer_index": 31, "normalized_entropy": 1.0000000186840277, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.6094379425048828, "group": 5, "latest_source_mass": 0.20000000298023224, "layer": 17, "max_source_mass": 0.20000000298023224, "mixer_index": 32, "normalized_entropy": 1.0000000186840277, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_4", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 17, "max_source_mass": 0.1666666716337204, "mixer_index": 33, "normalized_entropy": 1.0000001451965654, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 18, "max_source_mass": 0.1666666716337204, "mixer_index": 34, "normalized_entropy": 1.0000001451965654, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 18, "max_source_mass": 0.1666666716337204, "mixer_index": 35, "normalized_entropy": 1.0000001451965654, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 19, "max_source_mass": 0.1666666716337204, "mixer_index": 36, "normalized_entropy": 1.0000001451965654, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 19, "max_source_mass": 0.1666666716337204, "mixer_index": 37, "normalized_entropy": 1.0000001451965654, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 20, "max_source_mass": 0.1666666716337204, "mixer_index": 38, "normalized_entropy": 1.0000001451965654, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 20, "max_source_mass": 0.1666666716337204, "mixer_index": 39, "normalized_entropy": 1.0000001451965654, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.791759729385376, "group": 6, "latest_source_mass": 0.1666666716337204, "layer": 21, "max_source_mass": 0.1666666716337204, "mixer_index": 40, "normalized_entropy": 1.0000001451965654, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_5", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 21, "max_source_mass": 0.1428571492433548, "mixer_index": 41, "normalized_entropy": 1.00000003408302, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 22, "max_source_mass": 0.1428571492433548, "mixer_index": 42, "normalized_entropy": 1.00000003408302, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 22, "max_source_mass": 0.1428571492433548, "mixer_index": 43, "normalized_entropy": 1.00000003408302, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 23, "max_source_mass": 0.1428571492433548, "mixer_index": 44, "normalized_entropy": 1.00000003408302, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 23, "max_source_mass": 0.1428571492433548, "mixer_index": 45, "normalized_entropy": 1.00000003408302, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 24, "max_source_mass": 0.1428571492433548, "mixer_index": 46, "normalized_entropy": 1.00000003408302, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 24, "max_source_mass": 0.1428571492433548, "mixer_index": 47, "normalized_entropy": 1.00000003408302, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9459102153778076, "group": 7, "latest_source_mass": 0.1428571492433548, "layer": 25, "max_source_mass": 0.1428571492433548, "mixer_index": 48, "normalized_entropy": 1.00000003408302, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_6", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 25, "max_source_mass": 0.125, "mixer_index": 49, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 26, "max_source_mass": 0.125, "mixer_index": 50, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 26, "max_source_mass": 0.125, "mixer_index": 51, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 27, "max_source_mass": 0.125, "mixer_index": 52, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 27, "max_source_mass": 0.125, "mixer_index": 53, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 28, "max_source_mass": 0.125, "mixer_index": 54, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 28, "max_source_mass": 0.125, "mixer_index": 55, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.079441547393799, "group": 8, "latest_source_mass": 0.125, "layer": 29, "max_source_mass": 0.125, "mixer_index": 56, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_7", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 29, "max_source_mass": 0.1111111119389534, "mixer_index": 57, "normalized_entropy": 1.0000000180537645, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 30, "max_source_mass": 0.1111111119389534, "mixer_index": 58, "normalized_entropy": 1.0000000180537645, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 30, "max_source_mass": 0.1111111119389534, "mixer_index": 59, "normalized_entropy": 1.0000000180537645, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 31, "max_source_mass": 0.1111111119389534, "mixer_index": 60, "normalized_entropy": 1.0000000180537645, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 31, "max_source_mass": 0.1111111119389534, "mixer_index": 61, "normalized_entropy": 1.0000000180537645, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 32, "max_source_mass": 0.1111111119389534, "mixer_index": 62, "normalized_entropy": 1.0000000180537645, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 32, "max_source_mass": 0.1111111119389534, "mixer_index": 63, "normalized_entropy": 1.0000000180537645, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "output", "entropy_mean": 2.1972246170043945, "group": null, "latest_source_mass": 0.1111111119389534, "layer": null, "max_source_mass": 0.1111111119389534, "mixer_index": null, "normalized_entropy": 1.0000000180537645, "offset": null, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_8", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 } ], "mode": "uniform_groups_6_7_only", "optimizer_state_after": "b82a349700eda58e2330a22ca2ee1f7fb1406efd47fd9aa306122b368a114192", "optimizer_state_before": "b82a349700eda58e2330a22ca2ee1f7fb1406efd47fd9aa306122b368a114192", "optimizer_state_unchanged": true, "positions": { "attention_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "676759a2d7c0615ad7e1f246e90927fb829d00c8b6c3745032ae6b27ab7d902a", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.0022265110164880753, 0.0021613112185150385, 0.0025601957459002733, 0.0025323182344436646, 0.0027253651060163975, 0.002904455177485943, 0.0029208383057266474, 0.00299624796025455, 0.0030937574338167906, 0.0033804215490818024, 0.0034370317589491606, 0.0038969130255281925, 0.004255065228790045, 0.004119740333408117, 0.003970506135374308, 0.004385598469525576, 0.0041999719105660915, 0.005232699681073427, 0.004664790350943804, 0.004898969549685717, 0.005131533369421959, 0.005069608334451914, 0.0051318020559847355, 0.0058372304774820805, 0.0059690922498703, 0.00554921617731452, 0.0058550527319312096, 0.006255472078919411, 0.006502739619463682, 0.006122888531535864, 0.007071305997669697, 0.006241494789719582 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.00015962477027642308, "first_to_last_ratio": 3.7699040072151497, "imbalance_abs_log_ratio": 1.3270495388586014, "last_quartile_mean": 4.234186599205714e-05, "mean": 8.617327489446325e-05, "normalized": [ 2.5428430303860874, 2.3222064178412967, 2.1011513502014654, 1.8911470095267005, 1.7004447966680725, 1.5353700031893112, 1.4229335846259188, 1.3028620492588128, 1.2049362956920768, 1.1705197150696323, 1.0773612761873523, 1.0133200843661747, 0.9906112963967714, 0.9272530070793641, 0.8550207003106389, 0.8185600560490035, 0.7703816559177782, 0.7173918665339755, 0.6760157258502337, 0.6514535263006792, 0.6184138115448249, 0.6053606047045522, 0.5838821473278404, 0.5697017460729616, 0.5457761369570343, 0.5276086337508958, 0.5104655265174319, 0.4951752820428687, 0.47527333481689216, 0.465789239220051, 0.45896414036514044, 0.45180594922816086 ], "peak_layer": 1, "peak_normalized": 2.5428430303860874, "population_cv": 0.5705341151299013, "rest_mean": 9.280180475562466e-05, "spike_contrast": 0.5428688997682254, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.0379213644191625e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00021912511147093028, 0.00020011213200632483, 0.00018106309289578348, 0.00016296633111778647, 0.00014653289690613747, 0.00013230786134954542, 0.0001226188469445333, 0.00011227188952034339, 0.0001038333066389896, 0.00010086751717608422, 9.283974941354245e-05, 8.732111018616706e-05, 8.536421955795959e-05, 7.990442827576771e-05, 7.367993384832516e-05, 7.053800072753802e-05, 6.63863102090545e-05, 6.182000652188435e-05, 5.825448897667229e-05, 5.6137883802875876e-05, 5.329074338078499e-05, 5.216590579948388e-05, 5.031503678765148e-05, 4.909306517220102e-05, 4.703131708083674e-05, 4.5465763832908124e-05, 4.3988486140733585e-05, 4.2670875700423494e-05, 4.095585973118432e-05, 4.013858415419236e-05, 3.955044303438626e-05, 3.893359826179221e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.00021442661636683624, "first_to_last_ratio": 1.843144421770586, "imbalance_abs_log_ratio": 0.6114730379455638, "last_quartile_mean": 0.0001163373926829081, "mean": 0.00015221274247778638, "normalized": [ 1.7741446897870732, 1.648461275666186, 1.5301464224684427, 1.4217121808823485, 1.3265065496562456, 1.2454420544341354, 1.1897998831173078, 1.1336248235930768, 1.0869089599614004, 1.0643003125599004, 1.0227030055526172, 0.9941765648180964, 0.9788958472869692, 0.9511102301475627, 0.9228060651723192, 0.9052418824754537, 0.8846484396889256, 0.865292475366091, 0.8497311645646503, 0.8388691935254817, 0.8256303537004397, 0.8176325372729673, 0.8076852907347736, 0.8000670639426607, 0.7892588692968658, 0.7814415503461742, 0.7731202151361662, 0.7664994567889658, 0.758490072461763, 0.7529625284402992, 0.7485174912671357, 0.7441725498875049 ], "peak_layer": 1, "peak_normalized": 1.7741446897870732, "population_cv": 0.27364872432853743, "rest_mean": 0.00015762320577373935, "spike_contrast": 0.7803181014868814, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012299624067964033, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00027004742878489196, 0.0002509168116375804, 0.0002329077833564952, 0.00021640271006617695, 0.00020191119983792305, 0.00018957215070258826, 0.0001811027032090351, 0.00017255214333999902, 0.00016544139361940324, 0.00016200006939470768, 0.00015566842921543866, 0.0001513263414381072, 0.00014900042151566595, 0.0001447710965294391, 0.00014046284195501357, 0.0001377893495373428, 0.00013465476513374597, 0.00013170854072086513, 0.00012933991092722863, 0.0001276865805266425, 0.00012567146040964872, 0.00012445409083738923, 0.00012293999316170812, 0.00012178040196886286, 0.00012013525702059269, 0.00011894536146428436, 0.00011767874821089208, 0.00011667098442558199, 0.00011545185407157987, 0.00011461049143690616, 0.00011393390013836324, 0.00011327254469506443 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.19015566632151604, "first_to_last_ratio": 1.8431444167877922, "imbalance_abs_log_ratio": 0.6114730352421437, "last_quartile_mean": 0.10316916275769472, "mean": 0.13498378219082952, "normalized": [ 1.7741447141792424, 1.6484611693507762, 1.5301464164964773, 1.421712099711509, 1.3265065258282553, 1.245442086656075, 1.1897999853143832, 1.1336247998774232, 1.0869090145314644, 1.0643003539468234, 1.0227030148964142, 0.9941765578539046, 0.978895844117059, 0.9511102293531465, 0.9228060470726628, 0.9052419251832852, 0.8846484748200404, 0.8652925213936268, 0.8497311905993837, 0.8388692019864461, 0.8256303029992552, 0.8176324960789627, 0.8076852727084025, 0.800067049482784, 0.789258877083647, 0.7814415614650243, 0.7731201949085273, 0.7664994759042474, 0.758490077799444, 0.7529625178867759, 0.7485174642151933, 0.7441725362993381 ], "peak_layer": 1, "peak_normalized": 1.7741447141792424, "population_cv": 0.2736487184871982, "rest_mean": 0.13978183573042904, "spike_contrast": 0.7803180757143806, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.10907429307699204, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.23948076367378235, 0.2225155234336853, 0.20654495060443878, 0.1919080764055252, 0.17905686795711517, 0.16811448335647583, 0.16060370206832886, 0.1530209630727768, 0.14671508967876434, 0.14366328716278076, 0.13804832100868225, 0.13419771194458008, 0.1321350634098053, 0.12838445603847504, 0.12456385046243668, 0.12219297885894775, 0.11941319704055786, 0.116800457239151, 0.11469992995262146, 0.11323373764753342, 0.11144670099020004, 0.11036712676286697, 0.10902441293001175, 0.10799607634544373, 0.10653714835643768, 0.10548193752765656, 0.10435868799686432, 0.1034649983048439, 0.10238385945558548, 0.10163772851228714, 0.10103771835565567, 0.10045122355222702 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 6.883365585963475e-05, "first_to_last_ratio": 2.2497546817252747, "imbalance_abs_log_ratio": 0.8108211799277664, "last_quartile_mean": 3.059607183786284e-05, "mean": 4.691841348858361e-05, "normalized": [ 1.7359303646705315, 1.6475859007944922, 1.5536416135247637, 1.4921074478374423, 1.436565604934822, 1.343834173349448, 1.2787439414688093, 1.2483315192759759, 1.191227587726315, 1.1603086071531024, 1.0995306735006558, 1.059977946939621, 1.0583984121521266, 1.0176934649371145, 0.9754909410255406, 0.9631621800279858, 0.9242301483750994, 0.8752327792179452, 0.8454160845678106, 0.8232909667825862, 0.7863453139125789, 0.77746685672799, 0.7497386589361914, 0.7388508716017261, 0.7093180450413769, 0.6908354487677191, 0.6685471901846501, 0.6592354484095886, 0.631440334976849, 0.6235901145414114, 0.6199506557967245, 0.6139807028410055 ], "peak_layer": 1, "peak_normalized": 1.7359303646705315, "population_cv": 0.3220412433027962, "rest_mean": 4.907019663530971e-05, "spike_contrast": 0.7193528234378936, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.529878449626267e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 8.144709863699973e-05, 7.730211655143648e-05, 7.289439963642508e-05, 7.000731420703232e-05, 6.740137905580923e-05, 6.305056740529835e-05, 5.999663699185476e-05, 5.856973439222202e-05, 5.5890508519951254e-05, 5.4439839004771784e-05, 5.1588234782684594e-05, 4.9732483603293076e-05, 4.965837433701381e-05, 4.774856279254891e-05, 4.576848732540384e-05, 4.519004141911864e-05, 4.3363412260077894e-05, 4.106453343410976e-05, 3.966558142565191e-05, 3.862750600092113e-05, 3.6894074582960457e-05, 3.647751145763323e-05, 3.517654840834439e-05, 3.4665710700210184e-05, 3.328007733216509e-05, 3.241290323785506e-05, 3.136717350571416e-05, 3.0930281354812905e-05, 2.9626178729813546e-05, 2.925785884144716e-05, 2.9087101211189292e-05, 2.8807000489905477e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 0.0001734125216898974, "first_to_last_ratio": 1.495517223041696, "imbalance_abs_log_ratio": 0.4024721155990803, "last_quartile_mean": 0.00011595488103921525, "mean": 0.00013990532806928968, "normalized": [ 1.3768889945483427, 1.3340940553978193, 1.2910049686488183, 1.2527203273458163, 1.2147536525239127, 1.1769854337196524, 1.14744939519443, 1.1220955988146075, 1.0944022472959734, 1.0773022658134643, 1.0508559212107247, 1.03206209473486, 1.0208171898483585, 1.000561882500425, 0.9805161605170362, 0.9663129296065233, 0.9487424991891081, 0.931931984738623, 0.9181974345490227, 0.9075104531838204, 0.8943640941785878, 0.8862664021756573, 0.8757830774574916, 0.86790401991425, 0.8562894022841879, 0.8478155487572495, 0.8387227683262297, 0.8314011142413483, 0.8224071457687988, 0.8161875050523302, 0.8112199678685613, 0.8064334645939688 ], "peak_layer": 1, "peak_normalized": 1.3768889945483427, "population_cv": 0.16325957486062354, "rest_mean": 0.0001431148236776978, "spike_contrast": 0.8564734849544942, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001225740517838858, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0001926341064972803, 0.00018664686649572104, 0.00018061847367789596, 0.00017526224837638438, 0.0001699505082797259, 0.00016466653323732316, 0.00016053428407758474, 0.00015698715287726372, 0.00015311270544771105, 0.00015072032692842185, 0.00014702034241054207, 0.00014439098595175892, 0.00014281776384450495, 0.00013998393842484802, 0.00013717943511437625, 0.00013519232743419707, 0.00013273413060232997, 0.00013038225006312132, 0.00012846071331296116, 0.00012696554767899215, 0.0001251263020094484, 0.00012399339175317436, 0.0001225267187692225, 0.00012142439663875848, 0.00011979944974882528, 0.00011861391249112785, 0.00011734178406186402, 0.00011631744564510882, 0.00011505914153531194, 0.00011418898066040128, 0.00011349399574100971, 0.00011282433843007311 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 0.00014675117381557357, "first_to_last_ratio": 1.2676837465168667, "imbalance_abs_log_ratio": 0.23719141364257146, "last_quartile_mean": 0.00011576323686313117, "mean": 0.00013042020850662084, "normalized": [ 1.169988069421413, 1.156175152925158, 1.1417665260225645, 1.1306726331060184, 1.1183621029407895, 1.1062694829504214, 1.0940115056230622, 1.0844989956600726, 1.0711591662817939, 1.0579765491085218, 1.0443900226505625, 1.035471661013805, 1.0260498632741444, 1.013992390086173, 1.0062479316494695, 0.9977001177294099, 0.9863163473759529, 0.9769959732660659, 0.9687585669786032, 0.9602628595878113, 0.9503845997255937, 0.9426104617668664, 0.9330243108845692, 0.9259760938667744, 0.914797625470175, 0.906289421438531, 0.8956357003805738, 0.888470885240542, 0.8806216558595509, 0.8756991509238615, 0.8712858841028995, 0.8681382926882502 ], "peak_layer": 1, "peak_normalized": 1.169988069421413, "population_cv": 0.09123006364857297, "rest_mean": 0.00013202972385678786, "spike_contrast": 0.9219804606102014, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012172882561571896, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00015259008796419948, 0.00015078860451467335, 0.00014890942838974297, 0.0001474625605624169, 0.00014585701865144074, 0.00014427989663090557, 0.00014268120867200196, 0.00014144058513920754, 0.00013970080181024969, 0.0001379815221298486, 0.00013620956451632082, 0.00013504642993211746, 0.0001338176371064037, 0.00013224509893916547, 0.00013123506505507976, 0.0001301202573813498, 0.00012863558367826045, 0.00012742001854348928, 0.00012634569429792464, 0.00012523768236860633, 0.00012394935765769333, 0.00012293545296415687, 0.00012168522516731173, 0.00012076599523425102, 0.00011930809705518186, 0.00011819845531135798, 0.00011680899478960782, 0.00011587455810513347, 0.00011485085997264832, 0.00011420886585256085, 0.00011363328667357564, 0.00011322277714498341 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00030002388848515693, "first_to_last_ratio": 2.2836575149168885, "imbalance_abs_log_ratio": 0.8257783308640597, "last_quartile_mean": 0.0001313786706305109, "mean": 0.0001947244400071213, "normalized": [ 1.8998833386658438, 1.7835645331795598, 1.6713073472446418, 1.5703972875779748, 1.4780250735507643, 1.3761669788219328, 1.3066447563403325, 1.2401013786965793, 1.1762106459027857, 1.150547637187856, 1.0768735950666006, 1.0264900078881174, 1.0140784159549234, 0.9666871251601796, 0.9096841069789114, 0.8754807117769676, 0.8421876125778792, 0.8073809902734768, 0.77911667677541, 0.7607651063686129, 0.7403822030704592, 0.7308855611321243, 0.7152791523187013, 0.7043381880055872, 0.6944157299089508, 0.686864629891504, 0.680599049276497, 0.6764772714226681, 0.6702412094778135, 0.6672258960044359, 0.6635863560708457, 0.6581114274310631 ], "peak_layer": 1, "peak_normalized": 1.8998833386658438, "population_cv": 0.3592853832280775, "rest_mean": 0.00020492727344390005, "spike_contrast": 0.6813594750078023, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013962913944851608, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0003699537192005664, 0.0003473036049399525, 0.0003254443872720003, 0.00030579473241232336, 0.00028780760476365685, 0.00026797334430739284, 0.0002544356684666127, 0.0002414780465187505, 0.0002290369593538344, 0.00022403974435292184, 0.00020969360775779933, 0.00019988269195891917, 0.0001974658516701311, 0.00018823760910890996, 0.00017713772831484675, 0.00017047749133780599, 0.00016399451124016196, 0.0001572168112033978, 0.00015171305858530104, 0.00014813955931458622, 0.00014417050988413393, 0.00014232128160074353, 0.00013928233238402754, 0.0001371518592350185, 0.00013521971413865685, 0.00013374933041632175, 0.00013252926873974502, 0.00013172665785532445, 0.00013051234418526292, 0.00012992518895771354, 0.00012921648158226162, 0.000128150379168801 ] } } }, "mlp_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "afe29f3ef4d2992353f0bb59fe7c5c2c5050fca671a53184825a8d63cd83a2bc", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.0027073738165199757, 0.002722575329244137, 0.0026850115973502398, 0.0026861224323511124, 0.00271838391199708, 0.0026645169127732515, 0.0026859452482312918, 0.0027359756641089916, 0.0026767742820084095, 0.0027451682835817337, 0.0026999011170119047, 0.002663193503394723, 0.002626648638397455, 0.0026531771291047335, 0.0026517712976783514, 0.002676903735846281, 0.002670086920261383, 0.0026355870068073273, 0.0026610493659973145, 0.0026388936676084995, 0.0026469926815479994, 0.00264862016774714, 0.002641494618728757, 0.002674198243767023, 0.002557571977376938, 0.00252733426168561, 0.0025468147359788418, 0.0026856842450797558, 0.002551089273765683, 0.0025778315030038357, 0.0026001501828432083, 0.0025662274565547705 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.00015858857113926206, "first_to_last_ratio": 3.757867690304466, "imbalance_abs_log_ratio": 1.3238516928952826, "last_quartile_mean": 4.220174423608114e-05, "mean": 8.573221236929385e-05, "normalized": [ 2.5461641896458365, 2.3032502265274117, 2.0924715218207024, 1.8886722264368014, 1.7031663195050915, 1.537434754996766, 1.4223955235407997, 1.3049501874408618, 1.2055297161045848, 1.1692542365736316, 1.0769571962730058, 1.0144505847411098, 0.9913601784485884, 0.9317001386730132, 0.857757082878185, 0.8155634736467603, 0.7682612945963089, 0.7217806418748633, 0.678338533281248, 0.6511820384364709, 0.6192882306865346, 0.6053793146733001, 0.5848721334223365, 0.5718144052517429, 0.5462281883927541, 0.527515083157495, 0.5112993585545178, 0.496920907994256, 0.47722358366421574, 0.46552107689700334, 0.4601234027970099, 0.45317424906679377 ], "peak_layer": 1, "peak_normalized": 2.5461641896458365, "population_cv": 0.5686091737198813, "rest_mean": 9.231269224848668e-05, "spike_contrast": 0.5437781067692293, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.0197621021652594e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00021828828903380781, 0.00019746273756027222, 0.00017939221288543195, 0.0001619200484128669, 0.00014601621660403907, 0.000131807682919316, 0.00012194511509733275, 0.00011187626660102978, 0.00010335272963857278, 0.00010024275252362713, 9.232992306351662e-05, 8.697109296917915e-05, 8.499150135321543e-05, 7.987671415321529e-05, 7.353741239057854e-05, 6.992006092332304e-05, 6.586474046343938e-05, 6.187985127326101e-05, 5.8155463193543255e-05, 5.582727681030519e-05, 5.309295011102222e-05, 5.1900507969548926e-05, 5.0142381951445714e-05, 4.902291402686387e-05, 4.682935104938224e-05, 4.522503513726406e-05, 4.3834825191879645e-05, 4.260212881490588e-05, 4.0913433622336015e-05, 3.991015182691626e-05, 3.944739728467539e-05, 3.885163096128963e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.00021242323236947414, "first_to_last_ratio": 1.8345651540958352, "imbalance_abs_log_ratio": 0.6068074801791611, "last_quartile_mean": 0.0001157894184871111, "mean": 0.00015111673860701558, "normalized": [ 1.771707915810251, 1.6366668487510905, 1.5242909365839221, 1.4186705636795862, 1.3264449439106374, 1.2453857912414694, 1.1893415085703798, 1.1330085309323608, 1.0858463851390703, 1.062861239127963, 1.023237806768965, 0.9942126959777651, 0.978436257513136, 0.9531027445378829, 0.9232414968561398, 0.904013237249231, 0.8853704564278231, 0.8680164030054869, 0.8515402788738432, 0.8395817783841374, 0.8267435189825344, 0.8185143642831039, 0.809172703595369, 0.8007918347797284, 0.7909896391756025, 0.7821052393452608, 0.7753388667989797, 0.7682453290925442, 0.7604995795745633, 0.7549901567959042, 0.7508572350862155, 0.7467737131490535 ], "peak_layer": 1, "peak_normalized": 1.771707915810251, "population_cv": 0.2714170004285523, "rest_mean": 0.00015645500981783562, "spike_contrast": 0.7816309251520459, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012229007406858728, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0002677347220014781, 0.00024732775636948645, 0.00023034587502479553, 0.00021438486874103546, 0.00020044803386554122, 0.0001881986390799284, 0.00017972940986510366, 0.00017121655400842428, 0.00016408956435043365, 0.00016061612404882908, 0.0001546283601783216, 0.00015024218009784818, 0.00014785809617023915, 0.0001440297783119604, 0.00013951724395155907, 0.000136611532070674, 0.0001337942958343774, 0.00013117180787958205, 0.0001286819897359237, 0.00012687486014328897, 0.00012493478425312787, 0.00012369122123345733, 0.00012227953993715346, 0.00012101305037504062, 0.00011953177454415709, 0.00011818919301731512, 0.00011716668086592108, 0.00011609472858253866, 0.00011492421617731452, 0.00011409165017539635, 0.00011346709652571008, 0.00011285000800853595 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.18837904930114746, "first_to_last_ratio": 1.8345651581276443, "imbalance_abs_log_ratio": 0.6068074823768531, "last_quartile_mean": 0.10268321540206671, "mean": 0.13401183625683188, "normalized": [ 1.7717078885231814, 1.6366668119471057, 1.524290957183596, 1.418670514273678, 1.326444920903592, 1.2453858718827995, 1.1893415500975306, 1.1330085716119305, 1.0858463365841577, 1.062861212873729, 1.0232377434953237, 0.9942127340740732, 0.9784362443707608, 0.9531027271661117, 0.9232414917376033, 0.904013241653558, 0.885370477646398, 0.8680164390463408, 0.8515402681847718, 0.8395816969955481, 0.8267435345782925, 0.8185144279720658, 0.8091727254708025, 0.8007918405918669, 0.7909896328038464, 0.7821052665924377, 0.7753388466443417, 0.7682453528580236, 0.7604996016741158, 0.7549901615676585, 0.7508572336421532, 0.7467736753526057 ], "peak_layer": 1, "peak_normalized": 1.7717078885231814, "population_cv": 0.27141699472708714, "rest_mean": 0.1387458680956452, "spike_contrast": 0.7816309473985974, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.10844806432724, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.23742982745170593, 0.2193327248096466, 0.2042730301618576, 0.19011864066123962, 0.1777593195438385, 0.16689644753932953, 0.15938584506511688, 0.151836559176445, 0.1455162614583969, 0.1424359828233719, 0.13712596893310547, 0.13323627412319183, 0.13112203776836395, 0.12772704660892487, 0.1237252876162529, 0.1211484745144844, 0.11865012347698212, 0.11632447689771652, 0.11411647498607635, 0.11251388490200043, 0.11079341918230057, 0.10969062149524689, 0.10843872278928757, 0.10731558501720428, 0.10600197315216064, 0.10481136292219162, 0.10390458256006241, 0.1029539704322815, 0.10191594809293747, 0.10117761790752411, 0.10062375664710999, 0.10007651150226593 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 6.873187476230669e-05, "first_to_last_ratio": 2.248757500145603, "imbalance_abs_log_ratio": 0.8103778415280068, "last_quartile_mean": 3.0564378221242805e-05, "mean": 4.685761763312257e-05, "normalized": [ 1.7400006587426948, 1.6405194725375354, 1.5551306973978087, 1.4944078341288562, 1.4327724019957897, 1.3414187773842097, 1.2804088050749736, 1.2499327782691996, 1.1927955171655855, 1.1608931065609767, 1.0981756972550076, 1.0577423904261343, 1.0542786038595535, 1.0218640087393278, 0.9775807389332813, 0.9570972402588515, 0.9209982861589985, 0.8815092249082567, 0.8478232134430647, 0.8204952153707635, 0.7871780668180614, 0.7759097745574741, 0.7506963524238334, 0.7421155343193898, 0.7072927325438048, 0.6891814936654451, 0.6695399899996352, 0.6616343983211914, 0.6345111343049281, 0.6212687927046295, 0.6200231913288072, 0.6148038704019305 ], "peak_layer": 1, "peak_normalized": 1.7400006587426948, "population_cv": 0.3217536161653951, "rest_mean": 4.900405669845727e-05, "spike_contrast": 0.719671983430414, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.526684668031521e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 8.153228554874659e-05, 7.687083416385576e-05, 7.286971958819777e-05, 7.002439087955281e-05, 6.71363013680093e-05, 6.285568815656006e-05, 5.999690620228648e-05, 5.8568872191244736e-05, 5.5891556257847697e-05, 5.439668530016206e-05, 5.145789691596292e-05, 4.9563288484932855e-05, 4.9400983698433265e-05, 4.7882112994557247e-05, 4.580710447044112e-05, 4.484729652176611e-05, 4.3155785533599555e-05, 4.130542220082134e-05, 3.972697595600039e-05, 3.844645107164979e-05, 3.688528886414133e-05, 3.635728353401646e-05, 3.517584264045581e-05, 3.477376594673842e-05, 3.314205241622403e-05, 3.229340290999971e-05, 3.1373048841487616e-05, 3.10026116494555e-05, 2.9731680115219206e-05, 2.9111175535945222e-05, 2.9052809622953646e-05, 2.8808244678657502e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 0.0001719075116852764, "first_to_last_ratio": 1.4894635014227988, "imbalance_abs_log_ratio": 0.39841598895878805, "last_quartile_mean": 0.00011541572621354135, "mean": 0.00013894635094402474, "normalized": [ 1.3738985779819257, 1.3296082759737424, 1.2883384004275333, 1.2499940655466777, 1.2130730176151365, 1.1755503979222366, 1.147409009170091, 1.11990595257071, 1.0929037170303153, 1.075598370036652, 1.0512762916941902, 1.0312594730951, 1.0194418974669357, 1.0018996501730997, 0.9803460168062212, 0.9648004557293595, 0.9493010807852901, 0.9344788929945884, 0.9198953859218769, 0.908001147597105, 0.8952913731815082, 0.886953676542665, 0.8771894463102521, 0.8683888934041172, 0.8579399877904575, 0.8482799643655919, 0.8408636339472488, 0.8330870307470335, 0.82434015220157, 0.8181187963300758, 0.8135579418994486, 0.809009026741244 ], "peak_layer": 1, "peak_normalized": 1.3738985779819257, "population_cv": 0.16161423472023242, "rest_mean": 0.0001421073115866252, "spike_contrast": 0.8576417505420824, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012187716347398237, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0001908981939777732, 0.0001847442181315273, 0.00017900991952046752, 0.00017368211410939693, 0.00016855206922627985, 0.00016333843814209104, 0.00015942829486448318, 0.00015560684551019222, 0.0001518549834145233, 0.00014945046859793365, 0.00014607100456487387, 0.0001432897406630218, 0.00014164773165248334, 0.00013921030040364712, 0.000136215501697734, 0.0001340555027127266, 0.0001319019211223349, 0.00012984243221580982, 0.0001278161071240902, 0.00012616344611160457, 0.00012439746933523566, 0.00012323897681199014, 0.00012188227265141904, 0.00012065946793882176, 0.00011920763063244522, 0.00011786540562752634, 0.00011683493357850239, 0.00011575440294109285, 0.0001145390560850501, 0.00011367462138878182, 0.00011304090730845928, 0.00011240885214647278 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 0.00014552221000485588, "first_to_last_ratio": 1.262697055715921, "imbalance_abs_log_ratio": 0.2332499537229619, "last_quartile_mean": 0.00011524712863320019, "mean": 0.00012953460168318998, "normalized": [ 1.1676213631634658, 1.1525055688027706, 1.1391580058081985, 1.1289307982409096, 1.1166731577821778, 1.1054001772932092, 1.094813144753513, 1.0822854389705017, 1.0679308603491235, 1.0564894822224282, 1.0456838360716472, 1.0337072702557388, 1.0234945545465333, 1.0158819477934218, 1.0051772952892828, 0.9951467950359763, 0.9866643397779161, 0.9787559418439278, 0.970448736962688, 0.9607908684400951, 0.9509939230158972, 0.9428061355388626, 0.934924980051271, 0.9261034265480621, 0.9170582603044295, 0.9058562786860701, 0.8981778407016658, 0.8907564927732574, 0.8831168120837269, 0.8778216107141017, 0.8745192645108896, 0.8703053916682415 ], "peak_layer": 1, "peak_normalized": 1.1676213631634658, "population_cv": 0.08967236486753827, "rest_mean": 0.00013110875065817877, "spike_contrast": 0.9231588022206524, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012103419721825049, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00015124736819416285, 0.00014928934979252517, 0.000147560378536582, 0.00014623560127802193, 0.00014464781270362437, 0.00014318757166620344, 0.00014181618462316692, 0.00014019341324456036, 0.0001383339986205101, 0.00013685194426216185, 0.00013545223919209093, 0.00013390085950959474, 0.00013257795944809914, 0.00013159186346456409, 0.00013020524056628346, 0.00012890594371128827, 0.00012780717224813998, 0.00012678276107180864, 0.0001257066905964166, 0.0001244556624442339, 0.00012318661902099848, 0.0001221260172314942, 0.00012110513489460573, 0.0001199624384753406, 0.00011879077646881342, 0.00011733973224181682, 0.00011634510883595794, 0.00011538378748809919, 0.0001143941844929941, 0.00011370827269274741, 0.00011328050459269434, 0.0001127346622524783 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00029741139769612346, "first_to_last_ratio": 2.278072415841521, "imbalance_abs_log_ratio": 0.8233296537390045, "last_quartile_mean": 0.00013055396993877366, "mean": 0.00019328556300024502, "normalized": [ 1.889398576482247, 1.780233894901915, 1.6772594716902607, 1.566224958959574, 1.4762232610465842, 1.375571144634086, 1.3083686481730985, 1.2364401527267181, 1.173148897934763, 1.1508864184295666, 1.080457040307468, 1.0318582947152708, 1.0081957004611959, 0.9698994715426978, 0.9066127339785177, 0.8754550048258108, 0.8446146862114313, 0.8115724449181009, 0.7806677558079268, 0.7613480996239201, 0.7403134026279838, 0.7294052762839086, 0.7160186976339193, 0.7062575707090604, 0.6942777320753531, 0.6866005528973866, 0.6829938476675228, 0.6753403086482458, 0.6715967314167978, 0.6686860556999406, 0.6640286432895989, 0.6600445236791291 ], "peak_layer": 1, "peak_normalized": 1.889398576482247, "population_cv": 0.3579154260008552, "rest_mean": 0.00020340604748559632, "spike_contrast": 0.681567477924495, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013863494677934796, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0003651934675872326, 0.0003440935106482357, 0.00032419004128314555, 0.0003027286729775369, 0.0002853326441254467, 0.00026587804313749075, 0.00025288877077400684, 0.00023898603103589267, 0.00022675274522043765, 0.00022244972933549434, 0.0002088367473334074, 0.00019944331143051386, 0.00019486967357806861, 0.00018746756541077048, 0.00017523515271022916, 0.00016921281348913908, 0.0001632518251426518, 0.00015686523693148047, 0.00015089180669747293, 0.00014715759607497603, 0.00014309189282357693, 0.00014098350948188454, 0.00013839607709087431, 0.0001365093921776861, 0.0001341938623227179, 0.00013270997442305088, 0.00013201285037212074, 0.00013053353177383542, 0.0001298099523410201, 0.00012924736074637622, 0.00012834715016651899, 0.00012757707736454904 ] } } }, "post_attention_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "31853ffb07b24cc59e774bb20656840b4358b3080b42516d380f2dc0f46404c8", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.0022265110164880753, 0.0040087271481752396, 0.005379301495850086, 0.006410556845366955, 0.0027253651060163975, 0.004838948138058186, 0.006175771821290255, 0.00742313964292407, 0.0030937574338167906, 0.005150026176124811, 0.007019025273621082, 0.008143599145114422, 0.004255065228790045, 0.0066224560141563416, 0.008129785768687725, 0.010099578648805618, 0.0041999719105660915, 0.00719039561226964, 0.009030314162373543, 0.010736005380749702, 0.005131533369421959, 0.00787449348717928, 0.009912566281855106, 0.012113703414797783, 0.0059690922498703, 0.00876442901790142, 0.010806876234710217, 0.012353328987956047, 0.006502739619463682, 0.009470963850617409, 0.012123330496251583, 0.01384514570236206 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.00015962236011546338, "first_to_last_ratio": 3.7698422272252903, "imbalance_abs_log_ratio": 1.3270331510418396, "last_quartile_mean": 4.2341920561739244e-05, "mean": 8.61727822893954e-05, "normalized": [ 2.542734798705971, 2.3221448837042376, 2.101145461274622, 1.8911858524935854, 1.7004204056817789, 1.5354142426015858, 1.4229001770142031, 1.3028733810328659, 1.2049412416934622, 1.1705258152816296, 1.0773908232649334, 1.0133220774376726, 0.990609360100461, 0.9272534949358001, 0.8550385909364457, 0.8185651575045715, 0.7703972051466339, 0.7173947009730917, 0.6760256695667757, 0.6514523109028573, 0.618419077603279, 0.6053620388084041, 0.5838930841859182, 0.5697043695032278, 0.5457802700901755, 0.5276149005446632, 0.5104701332743111, 0.49517988582787786, 0.47527562953524777, 0.46579059316028215, 0.45896435764104826, 0.45181000957238127 ], "peak_layer": 1, "peak_normalized": 2.542734798705971, "population_cv": 0.5705202801130907, "rest_mean": 9.280119640472414e-05, "spike_contrast": 0.542873885449774, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.03793460666202e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00021911453222855926, 0.00020010568550787866, 0.00018106155039276928, 0.00016296874673571438, 0.00014652995741926134, 0.00013231091725174338, 0.0001226152671733871, 0.00011227222421439365, 0.00010383313929196447, 0.00010086746624438092, 9.284176485380158e-05, 8.732078276807442e-05, 8.536356472177431e-05, 7.99040135461837e-05, 7.368105434579775e-05, 7.053803710732609e-05, 6.638727063545957e-05, 6.181989738252014e-05, 5.825501284562051e-05, 5.613745815935545e-05, 5.329089253791608e-05, 5.216573117650114e-05, 5.0315691623836756e-05, 4.9093010602518916e-05, 4.703140439232811e-05, 4.546604395727627e-05, 4.398863165988587e-05, 4.267102849553339e-05, 4.095582335139625e-05, 4.013847137684934e-05, 3.955023566959426e-05, 3.893372559105046e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.00021442477373057045, "first_to_last_ratio": 1.8431254562763404, "imbalance_abs_log_ratio": 0.6114627481432037, "last_quartile_mean": 0.00011633759004325839, "mean": 0.00015221248349917005, "normalized": [ 1.7740805953138137, 1.6484457246979003, 1.5301434809496548, 1.4217241600875643, 1.3264958046529274, 1.2454577490392524, 1.1897861330408546, 1.1336265611690741, 1.086914920172498, 1.064301358569379, 1.0227209980571423, 0.9941753882589759, 0.9788979908211966, 0.9511107967643203, 0.9228143274453928, 0.9052447611165039, 0.8846530997418666, 0.8652930871753776, 0.8497381552743479, 0.8388656494639406, 0.8256334792987466, 0.8176309647342147, 0.8076951735882674, 0.8000661307359944, 0.7892605467734257, 0.78144613040264, 0.7731219607566042, 0.7664988488799842, 0.7584935618388989, 0.7529639051546407, 0.7485195296370439, 0.7441790263875565 ], "peak_layer": 1, "peak_normalized": 1.7740805953138137, "population_cv": 0.2736411578809574, "rest_mean": 0.00015762286892384981, "spike_contrast": 0.7803207938393821, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012299640220589936, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00027003721334040165, 0.00025091401766985655, 0.00023290693934541196, 0.0002164041652576998, 0.000201909220777452, 0.0001895742170745507, 0.00018110030214302242, 0.00017255211423616856, 0.00016544201935175806, 0.00016199995297938585, 0.00015567090304102749, 0.00015132590488065034, 0.0001490004942752421, 0.00014477093645837158, 0.00014046386058907956, 0.00013778955326415598, 0.0001346552453469485, 0.00013170840975362808, 0.00012934075493831187, 0.00012768582382705063, 0.00012567172234412283, 0.00012445363972801715, 0.00012294128828216344, 0.00012178005272289738, 0.00012013530795229599, 0.00011894585622940212, 0.00011767881369451061, 0.00011667069338727742, 0.00011545218876563013, 0.00011461050598882139, 0.00011393401655368507, 0.00011327333777444437 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.190154030919075, "first_to_last_ratio": 1.84312545379361, "imbalance_abs_log_ratio": 0.6114627467961817, "last_quartile_mean": 0.10316933691501617, "mean": 0.13498355145566165, "normalized": [ 1.774080628234443, 1.6484456620184311, 1.5301434020504485, 1.4217242444591272, 1.326495766996536, 1.2454576834375484, 1.189786122599316, 1.133626627255238, 1.0869149569673349, 1.0643014004698326, 1.0227209907477024, 0.9941753870542451, 0.9788979589698585, 0.9511107512164827, 0.9228143032197887, 0.9052447972745712, 0.8846531331847195, 0.8652931173486506, 0.8497382179112226, 0.8388656682549475, 0.8256334805754935, 0.8176309131099365, 0.8076951535479161, 0.8000661540388709, 0.7892605573866218, 0.7814461538045975, 0.7731219580171443, 0.7664988542586116, 0.7584935821787124, 0.7529638601652359, 0.7485195164462104, 0.7441789968002049 ], "peak_layer": 1, "peak_normalized": 1.774080628234443, "population_cv": 0.27364115345546325, "rest_mean": 0.13978153549962574, "spike_contrast": 0.7803207857775153, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.10907443761825561, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.2394717037677765, 0.22251304984092712, 0.2065441906452179, 0.19190938770771027, 0.1790551096200943, 0.16811630129814148, 0.16060155630111694, 0.1530209481716156, 0.14671564102172852, 0.1436631828546524, 0.13805051147937775, 0.13419732451438904, 0.13213512301445007, 0.1283843070268631, 0.1245647519826889, 0.12219315767288208, 0.11941362172365189, 0.11680033802986145, 0.11470068246126175, 0.1132330670952797, 0.11144693940877914, 0.11036672443151474, 0.10902556031942368, 0.10799577087163925, 0.10653719305992126, 0.10548237711191177, 0.1043587476015091, 0.10346473753452301, 0.10238415747880936, 0.10163773596286774, 0.10103782266378403, 0.10045192390680313 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 6.883367768750759e-05, "first_to_last_ratio": 2.249759357551583, "imbalance_abs_log_ratio": 0.810823258297238, "last_quartile_mean": 3.059601795030176e-05, "mean": 4.691846504556452e-05, "normalized": [ 1.7359059710084281, 1.6475789727938472, 1.5536456441204052, 1.4921194549590373, 1.4365657321885223, 1.3438513058553838, 1.2787343947807137, 1.2483299149152907, 1.1912354282513808, 1.1603083401306369, 1.099547144002948, 1.059972440022979, 1.0583955432739494, 1.0176800955770247, 0.9754963047746087, 0.9631595708780456, 0.9242400656751482, 0.8752301116137658, 0.8454245377074081, 0.8232853322610043, 0.7863494898171388, 0.7774642190168813, 0.7497486129019063, 0.7388483538626245, 0.709315327139865, 0.6908343794807076, 0.6685469983108738, 0.6592365073812129, 0.6314341746588822, 0.6235898169926835, 0.6199474545603838, 0.6139783610863121 ], "peak_layer": 1, "peak_normalized": 1.7359059710084281, "population_cv": 0.3220406003070692, "rest_mean": 4.9070239684725595e-05, "spike_contrast": 0.7193541792517962, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.52988819940947e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 8.144604362314567e-05, 7.730187644483522e-05, 7.28946688468568e-05, 7.000795449130237e-05, 6.740145909134299e-05, 6.305144052021205e-05, 5.999625500408001e-05, 5.8569723478285596e-05, 5.589093780145049e-05, 5.4439886298496276e-05, 5.158906424185261e-05, 4.973227987647988e-05, 4.9658294301480055e-05, 4.774798799189739e-05, 4.576878927764483e-05, 4.51899686595425e-05, 4.3363925215089694e-05, 4.1064453398576006e-05, 3.9666021621087566e-05, 3.86272840842139e-05, 3.689431105158292e-05, 3.6477427784120664e-05, 3.5177054087398574e-05, 3.466563066467643e-05, 3.3279986382694915e-05, 3.2412888685939834e-05, 3.136719897156581e-05, 3.093036502832547e-05, 2.9625922252307646e-05, 2.9257877031341195e-05, 2.9086982976878062e-05, 2.8806922273361124e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 0.00017341253987979144, "first_to_last_ratio": 1.4955148931299893, "imbalance_abs_log_ratio": 0.4024705576674927, "last_quartile_mean": 0.00011595507385209203, "mean": 0.00013990553861731314, "normalized": [ 1.3768818258178377, 1.3340879911950096, 1.2910054180628048, 1.2527236427111486, 1.2147542166907819, 1.1769889670731195, 1.1474411155803916, 1.1220953663134672, 1.0944075691302455, 1.0773007485615413, 1.0508678613623372, 1.0320596054419082, 1.0208161736522385, 1.000559648637946, 0.9805197815181176, 0.9663128275349431, 0.948744191771126, 0.9319305822463463, 0.9182019814348666, 0.9075038868226692, 0.8943650364968407, 0.886262676120959, 0.8757907045344768, 0.8679002174812178, 0.856288373659771, 0.8478178092769747, 0.8387215581135022, 0.8313975747674917, 0.8224084044020752, 0.8161864847722404, 0.8112200992010112, 0.8064376596145924 ], "peak_layer": 1, "peak_normalized": 1.3768818258178377, "population_cv": 0.16325888721442564, "rest_mean": 0.0001431150389921472, "spike_contrast": 0.8564734877369171, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001225742365932092, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0001926333934534341, 0.00018664629897102714, 0.00018061880837194622, 0.0001752629759721458, 0.00016995084297377616, 0.0001646672753849998, 0.00016053336730692536, 0.00015698735660407692, 0.00015311368042603135, 0.00015072034148033708, 0.00014702223415952176, 0.00014439085498452187, 0.0001428178366040811, 0.00013998383656144142, 0.00013718014815822244, 0.00013519251660909504, 0.00013273456715978682, 0.00013038225006312132, 0.00012846154277212918, 0.00012696482008323073, 0.00012512662215158343, 0.0001239930570591241, 0.00012252797023393214, 0.000121424047392793, 0.00011979948612861335, 0.00011861440725624561, 0.00011734179133782163, 0.0001163171255029738, 0.00011505949078127742, 0.00011418900976423174, 0.00011349418491590768, 0.00011282509512966499 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 0.00014675183047074825, "first_to_last_ratio": 1.2676545313577883, "imbalance_abs_log_ratio": 0.23716836728277432, "last_quartile_mean": 0.00011576642282307148, "mean": 0.00013041823922321782, "normalized": [ 1.1700698937843759, 1.1560440994564782, 1.1417398043557874, 1.130463200949706, 1.118403537277814, 1.1065304334184665, 1.0939881912841023, 1.0846815122144253, 1.0709468270677047, 1.0579907390333845, 1.0442838370166418, 1.035550673160169, 1.025959021745254, 1.0140203095307772, 1.0062047700154308, 0.9976631870190885, 0.9861765922450596, 0.9770016877798172, 0.9687702939454803, 0.9602422119882037, 0.9504995944091904, 0.9425883202531964, 0.9330014667281973, 0.9259385264566372, 0.9147443798098084, 0.906202796813466, 0.8958065504175365, 0.8885655303249116, 0.8806194435723184, 0.8757905905296257, 0.8713783728577705, 0.8681336045391749 ], "peak_layer": 1, "peak_normalized": 1.1700698937843759, "population_cv": 0.09121429133712369, "rest_mean": 0.0001320278307604086, "spike_contrast": 0.921975648780332, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001217264449223876, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0001525984553154558, 0.0001507692359155044, 0.000148903694935143, 0.00014743302017450333, 0.00014586022007279098, 0.00014431175077334046, 0.00014267601363826543, 0.00014146225294098258, 0.000139670999487862, 0.00013798128929920495, 0.00013619365927297622, 0.00013505469541996717, 0.00013380376913119107, 0.00013224674330558628, 0.00013122745440341532, 0.00013011347618885338, 0.00012861541472375393, 0.00012741883983835578, 0.0001263453159481287, 0.0001252330985153094, 0.0001239624834852293, 0.00012293070903979242, 0.00012168040848337114, 0.00012075927224941552, 0.00011929935135412961, 0.00011818537313956767, 0.00011682951299007982, 0.00011588515189941972, 0.00011484883725643158, 0.00011421906674513593, 0.00011364363308530301, 0.00011322045611450449 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00030004180916876066, "first_to_last_ratio": 2.2837506327398005, "imbalance_abs_log_ratio": 0.8258191057718037, "last_quartile_mean": 0.00013138116082700435, "mean": 0.00019473525253488333, "normalized": [ 1.9000096511818072, 1.783700741635724, 1.6708066910504589, 1.570366884717585, 1.4783058799917985, 1.376185321596543, 1.3064973297630273, 1.2402700043336092, 1.1767953849206394, 1.1505341943958838, 1.0771542571420398, 1.026098162637436, 1.013898063759585, 0.9669473025647412, 0.9095398902231254, 0.8755205777387188, 0.842301214161826, 0.8073124727024382, 0.7791991818813049, 0.760574607765185, 0.7402055397447304, 0.7308143413325056, 0.7152667122155245, 0.7043714155678553, 0.6942467749765324, 0.6868749151283189, 0.680655564574882, 0.676363713518914, 0.6701061029149729, 0.6672276319278244, 0.6636177363814285, 0.6582317375530352 ], "peak_layer": 1, "peak_normalized": 1.9000096511818072, "population_cv": 0.35930746026536914, "rest_mean": 0.00020494150898971216, "spike_contrast": 0.6812747128050888, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013962146767880766, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0003699988592416048, 0.0003473494143690914, 0.0003253649629186839, 0.0003058057918678969, 0.0002878782688640058, 0.0002679917961359024, 0.0002544210874475539, 0.00024152429250534624, 0.00022916354646440595, 0.000224049566895701, 0.00020975990628357977, 0.00019981748482678086, 0.000197441695490852, 0.0001882987271528691, 0.00017711948021315038, 0.00017049472080543637, 0.00016402573965024203, 0.00015721219824627042, 0.00015173754945863038, 0.00014811068831477314, 0.00014414411270990968, 0.0001423153153154999, 0.0001392876438330859, 0.00013716594548895955, 0.00013519432104658335, 0.00013375876005738974, 0.00013254763325676322, 0.0001317118585575372, 0.00013049328117631376, 0.00012993274140171707, 0.00012922976748086512, 0.00012818092363886535 ] } } }, "post_mlp_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "7fd9c911e93175896ad618842c120e77b3f017c17b7eec9c2bfb01fc6dbc8ac2", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.0034885562490671873, 0.004831995815038681, 0.005952076520770788, 0.006925336085259914, 0.00385080324485898, 0.005542940925806761, 0.006729288958013058, 0.007927131839096546, 0.004081308841705322, 0.005811207927763462, 0.007534808944910765, 0.008581922389566898, 0.00499920267611742, 0.007102519739419222, 0.008542952127754688, 0.010447698645293713, 0.005041283555328846, 0.007636935915797949, 0.009387043304741383, 0.011020516976714134, 0.005807018838822842, 0.008367395959794521, 0.01024632342159748, 0.012378672137856483, 0.006453148555010557, 0.009127611294388771, 0.011037957854568958, 0.012611051090061665, 0.007010402157902718, 0.009794782847166061, 0.012445803731679916, 0.014087481424212456 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.00015858902861509705, "first_to_last_ratio": 3.757869257587208, "imbalance_abs_log_ratio": 1.3238521099622325, "last_quartile_mean": 4.220184837322449e-05, "mean": 8.57325238712292e-05, "normalized": [ 2.5462257183789148, 2.3032637538343064, 2.0924289533379263, 1.8886492391629932, 1.7031256747388024, 1.5374176267930122, 1.4224231993460965, 1.3049597038569594, 1.2055618292025094, 1.169245405308266, 1.0769812048488787, 1.0144334047898542, 0.9913512297333369, 0.9317090592952362, 0.8577499774884012, 0.81555804918997, 0.7682725064221458, 0.7217807351250844, 0.6783324192661732, 0.6511800118936032, 0.6192916242842709, 0.6053728292385717, 0.5848720876072855, 0.5718124973482859, 0.5462302773879167, 0.5275173250132701, 0.5112998770983732, 0.49691833866078555, 0.47721658788927634, 0.46551989467389665, 0.46012635628943105, 0.45317260249616614 ], "peak_layer": 1, "peak_normalized": 2.5462257183789148, "population_cv": 0.5686111305901927, "rest_mean": 9.231303704803361e-05, "spike_contrast": 0.5437775023084314, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.019775271648541e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00021829435718245804, 0.00019746461475733668, 0.0001793892151908949, 0.0001619186659809202, 0.00014601326256524771, 0.00013180669338908046, 0.00012194793089292943, 0.00011187748896190897, 0.00010335585830034688, 0.00010024235962191597, 9.233231685357168e-05, 8.69699360919185e-05, 8.499104296788573e-05, 7.987776916706935e-05, 7.353707042057067e-05, 6.991984992055222e-05, 6.586594099644572e-05, 6.188008410390466e-05, 5.8155150327365845e-05, 5.582730591413565e-05, 5.309343396220356e-05, 5.190014053368941e-05, 5.014256021240726e-05, 4.90229285787791e-05, 4.682970029534772e-05, 4.522539165918715e-05, 4.383502891869284e-05, 4.2602063331287354e-05, 4.0912982512963936e-05, 3.991019548266195e-05, 3.944779382436536e-05, 3.885163096128963e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.00021242368711682502, "first_to_last_ratio": 1.8345658968465297, "imbalance_abs_log_ratio": 0.6068078850437921, "last_quartile_mean": 0.0001157896194854402, "mean": 0.0001511171253696375, "normalized": [ 1.7717451736709768, 1.636674793183293, 1.5242755762025908, 1.4186515254985965, 1.3264266232459345, 1.2453738409548603, 1.1893512719369177, 1.1330135273991528, 1.085860457802012, 1.0628542818802846, 1.023252039669589, 0.9942070699734702, 0.9784308644775869, 0.9531083940389613, 0.9232390376562555, 0.9040072643238835, 0.8853775311228679, 0.8680181295617843, 0.8515367513400655, 0.8395795332993111, 0.8267439067367663, 0.8185133286645813, 0.8091730400239778, 0.800791325997512, 0.7909899739933552, 0.7821072339266707, 0.7753403490721149, 0.7682422073314642, 0.7604971517068638, 0.7549863467424895, 0.7508596466770359, 0.7467718018887739 ], "peak_layer": 1, "peak_normalized": 1.7717451736709768, "population_cv": 0.2714186609524326, "rest_mean": 0.00015645541295978345, "spike_contrast": 0.7816308178118705, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012229037238284946, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0002677410375326872, 0.0002473295899108052, 0.00023034414334688336, 0.00021438254043459892, 0.00020044577831868082, 0.00018819731485564262, 0.00017973134526982903, 0.000171217747265473, 0.0001640921109355986, 0.00016061548376455903, 0.00015463090676348656, 0.00015024171443656087, 0.0001478576596127823, 0.00014403100067283958, 0.00013951722939964384, 0.00013661097909789532, 0.00013379570737015456, 0.0001311724045081064, 0.0001286817860091105, 0.00012687484559137374, 0.0001249351626029238, 0.00012369138130452484, 0.00012227990373503417, 0.00012101328320568427, 0.00011953213106608018, 0.0001181897969217971, 0.0001171672047348693, 0.00011609455395955592, 0.00011492414341773838, 0.0001140913664130494, 0.00011346775136189535, 0.00011285000800853595 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.18837944976985455, "first_to_last_ratio": 1.8345658800697515, "imbalance_abs_log_ratio": 0.6068078758989703, "last_quartile_mean": 0.10268339328467846, "mean": 0.1340121782850474, "normalized": [ 1.7717451751532867, 1.6366747548098712, 1.5242756140248972, 1.4186514377480737, 1.3264266357207033, 1.2453739091726768, 1.1893513017871604, 1.1330134634090196, 1.0858604665419038, 1.0628542749024097, 1.0232520332438115, 0.9942070832344971, 0.9784308561813967, 0.9531084116998004, 0.9232390798282929, 0.9040072094634526, 0.8853775581660035, 0.8680181154203164, 0.8515367049614931, 0.8395794986042229, 0.8267439263823189, 0.8185133396785964, 0.809173050926764, 0.8007913534932437, 0.7909900046670136, 0.782107217826699, 0.7753403703776178, 0.7682422246066585, 0.7604971603471317, 0.7549863443945886, 0.7508596538022533, 0.7467717694238248 ], "peak_layer": 1, "peak_normalized": 1.7717451751532867, "population_cv": 0.27141865758333467, "rest_mean": 0.13874622461972413, "spike_contrast": 0.7816308398662988, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.10844832807779312, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.23743543028831482, 0.21933434903621674, 0.20427149534225464, 0.19011656939983368, 0.17775732278823853, 0.16689527034759521, 0.15938755869865417, 0.15183760225772858, 0.14551852643489838, 0.14243541657924652, 0.13712823390960693, 0.1332358568906784, 0.1311216503381729, 0.12772813439369202, 0.1237252801656723, 0.12114797532558441, 0.1186513751745224, 0.1163249984383583, 0.11411628872156143, 0.11251387745141983, 0.11079375445842743, 0.10969075560569763, 0.10843904316425323, 0.107315793633461, 0.10600229352712631, 0.104811891913414, 0.10390505194664001, 0.10295381397008896, 0.1019158810377121, 0.10117736458778381, 0.10062433779239655, 0.10007651150226593 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 6.873207303215167e-05, "first_to_last_ratio": 2.248757814140914, "imbalance_abs_log_ratio": 0.8103779811585758, "last_quartile_mean": 3.0564462122129044e-05, "mean": 4.6857780716891284e-05, "normalized": [ 1.7400119939228862, 1.6405409364325143, 1.555121247708426, 1.4944054279879166, 1.4327475398601994, 1.3414100714975594, 1.2804101716466747, 1.2499370459040815, 1.192804641985012, 1.1608858830017554, 1.098191983597808, 1.057744066133373, 1.0542717513613309, 1.021869458338179, 0.9775749297615711, 0.9570927445976433, 0.9210091333285576, 0.8815125232807631, 0.847820728512489, 0.8204970180441978, 0.7871838673851995, 0.7759041238100245, 0.7506938949785397, 0.7421070509174572, 0.7072987335050381, 0.6891811912847667, 0.6695407652827761, 0.6616333377893114, 0.6345027924930273, 0.6212629426015038, 0.6200302724102036, 0.614801730639213 ], "peak_layer": 1, "peak_normalized": 1.7400119939228862, "population_cv": 0.32175451551289735, "rest_mean": 4.9004235565748616e-05, "spike_contrast": 0.7196709452950106, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.5266924533061685e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 8.153310045599937e-05, 7.687210745643824e-05, 7.28695304132998e-05, 7.002452184678987e-05, 6.713537004543468e-05, 6.28554989816621e-05, 5.9997179050697014e-05, 5.8569276006892323e-05, 5.589217835222371e-05, 5.439653614303097e-05, 5.1458839152473956e-05, 4.9563539505470544e-05, 4.9400834541302174e-05, 4.788253500009887e-05, 4.58069916930981e-05, 4.4847241952084005e-05, 4.315644400776364e-05, 4.130572051508352e-05, 3.972699778387323e-05, 3.8446669350378215e-05, 3.688568904181011e-05, 3.635714529082179e-05, 3.5175849916413426e-05, 3.477348946034908e-05, 3.3142448955914006e-05, 3.2293501135427505e-05, 3.13731943606399e-05, 3.1002669857116416e-05, 2.9731392714893445e-05, 2.9111002731951885e-05, 2.905324254243169e-05, 2.8808244678657502e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 0.00017190765356644988, "first_to_last_ratio": 1.4894621368061003, "imbalance_abs_log_ratio": 0.3984150727783497, "last_quartile_mean": 0.00011541592721187044, "mean": 0.00013894661265112518, "normalized": [ 1.3739040544627474, 1.3296102750467884, 1.2883342981472405, 1.2499918159027226, 1.2130627732820631, 1.175544413473833, 1.1474083142393565, 1.1199112790671786, 1.092909618037032, 1.0755929927697745, 1.0512861461262244, 1.0312571117882559, 1.0194388253067554, 1.0019061415077835, 0.980344379776871, 0.9647961249933756, 0.9493075664595542, 0.9344817410273759, 0.9198929201794295, 0.9079992279088673, 0.8952920956888009, 0.8869530532606651, 0.8771902552778059, 0.8683889858357474, 0.8579407806493952, 0.8482823463732029, 0.8408658204630546, 0.8330843095877946, 0.8243382853592501, 0.818115422616924, 0.8135611224181157, 0.8090075029660142 ], "peak_layer": 1, "peak_normalized": 1.3739040544627474, "population_cv": 0.16161403917150835, "rest_mean": 0.00014210757001786045, "spike_contrast": 0.8576421569620596, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012187744287075474, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00019089931447524577, 0.00018474484386388212, 0.00017900968668982387, 0.00017368212866131216, 0.0001685509632807225, 0.00016333791427314281, 0.00015942849859129637, 0.00015560787869617343, 0.00015185608936008066, 0.00014945000293664634, 0.00014607264893129468, 0.0001432896824553609, 0.00014164757158141583, 0.0001392114645568654, 0.00013621553080156446, 0.0001340551534667611, 0.00013190307072363794, 0.00012984307250007987, 0.0001278160052606836, 0.00012616341700777411, 0.0001243978040292859, 0.00012323912233114243, 0.00012188261462142691, 0.00012065970804542303, 0.00011920796532649547, 0.00011786595860030502, 0.00011683545744745061, 0.00011575424287002534, 0.00011453901242930442, 0.00011367436673026532, 0.00011304156214464456, 0.00011240885214647278 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 0.00014552358334185556, "first_to_last_ratio": 1.262638843036328, "imbalance_abs_log_ratio": 0.2332038508028458, "last_quartile_mean": 0.00011525352965691127, "mean": 0.00012953498708156985, "normalized": [ 1.1676304712442283, 1.1525836984023852, 1.1391521450612143, 1.1288261090295841, 1.1167600441451297, 1.1054423860149858, 1.094731137320412, 1.082319740349084, 1.067957453000188, 1.0563935463552665, 1.04560972624621, 1.0337959762176818, 1.0235535208863231, 1.0159040893770892, 1.0050628637064216, 0.9950792389307415, 0.9865814183778941, 0.9786230528286792, 0.9704418054120744, 0.9608381133357455, 0.9508407831226554, 0.9429749854457838, 0.9346839821902475, 0.9262276149685831, 0.9170542960948607, 0.9059488475652709, 0.8984648923618161, 0.8908037775288885, 0.8831483920181569, 0.8778579808382188, 0.8744051093280881, 0.8703028022960919 ], "peak_layer": 1, "peak_normalized": 1.1676304712442283, "population_cv": 0.0896642712448568, "rest_mean": 0.00013110964479252557, "spike_contrast": 0.9231344927669946, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012103183544240892, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00015124899800866842, 0.00014929991448298097, 0.00014756005839444697, 0.00014622247545048594, 0.00014465949789155275, 0.00014319346519187093, 0.00014180598373059183, 0.00014019827358424664, 0.00013833785487804562, 0.00013683992438018322, 0.0001354430423816666, 0.00013391274842433631, 0.0001325859921053052, 0.00013159512309357524, 0.00013019080506637692, 0.00012889757636003196, 0.00012779681128449738, 0.00012676592450588942, 0.00012570616672746837, 0.00012446215259842575, 0.00012316714855842292, 0.00012214825255796313, 0.00012107427755836397, 0.00011997888213954866, 0.00011879061639774591, 0.00011735207226593047, 0.0001163826382253319, 0.0001153902558144182, 0.00011439861555118114, 0.00011371332220733166, 0.00011326605454087257, 0.0001127346622524783 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00029742828155576717, "first_to_last_ratio": 2.2781890121807993, "imbalance_abs_log_ratio": 0.8233808344454435, "last_quartile_mean": 0.00013055469935352448, "mean": 0.00019329176166138495, "normalized": [ 1.8891850069388627, 1.7801603926271052, 1.6773544463608039, 1.566500262900107, 1.476456430986117, 1.3754925511314788, 1.3084504582544147, 1.236424592515421, 1.172965600373877, 1.150871569148191, 1.0802294364746856, 1.0319209662753672, 1.0080063247818516, 0.9698985570693867, 0.9067931019504507, 0.8756163462587222, 0.8444435806568995, 0.8113939671049883, 0.7809677247391495, 0.7613442367276676, 0.7402582678385915, 0.7293562882295931, 0.7162426695419845, 0.7062419232587185, 0.6940340549797547, 0.6866100033591334, 0.6831688143117768, 0.6753320518823956, 0.6715919825156708, 0.6686410475337363, 0.6640239865195754, 0.6600233567535221 ], "peak_layer": 1, "peak_normalized": 1.8891850069388627, "population_cv": 0.357915029863326, "rest_mean": 0.00020341356920888993, "spike_contrast": 0.6815376252628051, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001386340009048581, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00036516389809548855, 0.0003440903383307159, 0.0003242187958676368, 0.0003027915954589844, 0.00028538686456158757, 0.00026587137836031616, 0.0002529126941226423, 0.00023899068764876574, 0.00022672458726447076, 0.00022245399304665625, 0.0002087994507746771, 0.00019946182146668434, 0.00019483931828290224, 0.00018747340072877705, 0.00017527563613839447, 0.00016924942610785365, 0.00016322398732881993, 0.00015683576930314302, 0.0001509546273155138, 0.00014716156874783337, 0.0001430858246749267, 0.0001409785618307069, 0.00013844380737282336, 0.00013651074550580233, 0.00013415106514003128, 0.00013271605712361634, 0.00013205090363044292, 0.00013053612201474607, 0.00012981319741811603, 0.00012924280599690974, 0.00012835036613978446, 0.00012757707736454904 ] } } }, "pre_attention_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "de71d4cf2d509a1063f6eac7f50c4885e9a058dee7e4b993ad40f5527f528d00", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.02802751213312149, 0.01413465291261673, 0.014225807972252369, 0.01433455292135477, 0.01440478302538395, 0.009703479707241058, 0.00977881159633398, 0.009858808480203152, 0.009950309991836548, 0.00752287870272994, 0.007571155205368996, 0.007693085819482803, 0.007788818795233965, 0.006353048607707024, 0.006389216054230928, 0.0064707002602517605, 0.00656962301582098, 0.005553483963012695, 0.0056249531917274, 0.005688921548426151, 0.005768724251538515, 0.004966245498508215, 0.005057469941675663, 0.005098348017781973, 0.005216081161051989, 0.0046758754178881645, 0.004727107938379049, 0.004777040798217058, 0.004837613552808762, 0.0044186655431985855, 0.004510571248829365, 0.0045849340967834 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.0001580166108396952, "first_to_last_ratio": 1.9713740353643698, "imbalance_abs_log_ratio": 0.6787307795042399, "last_quartile_mean": 8.015557068574708e-05, "mean": 0.00011843875881822896, "normalized": [ 0.8918679597819287, 1.5814819593588525, 1.5722239987443252, 1.3246067564285042, 1.1252650527356378, 1.512743589128775, 1.3788112428745087, 1.2863035124436846, 1.097095055843533, 1.468154353173571, 1.1523254606068964, 1.0839888571909224, 1.054518217028451, 1.146335080549274, 1.1665208527784754, 0.8931136212023528, 0.8203804970547371, 1.073741223277434, 0.919654742394771, 0.8223971330190447, 0.7711288002244273, 0.8470775332584999, 0.8298711629738247, 0.7662487463744663, 0.7210382462310316, 0.7071376059209309, 0.7189559392079661, 0.727980518763353, 0.6111916361855597, 0.6986829946379066, 0.6244678178312321, 0.6046898327751223 ], "peak_layer": 2, "peak_normalized": 1.5814819593588525, "population_cv": 0.2904715201464359, "rest_mean": 0.00012310891098523926, "spike_contrast": 0.7572151875143335, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.321993711637333e-05, "top_five_layers": [ 2, 3, 6, 10, 7 ], "top_five_spike_overlap": 0 }, "values": [ 0.00010563173418631777, 0.0001873087603598833, 0.00018621225899551064, 0.00015688478015363216, 0.0001332749961875379, 0.00017916747310664505, 0.00016330469225067645, 0.00015234819147735834, 0.00012993857671972364, 0.00017388637934345752, 0.0001364799973089248, 0.0001283862948184833, 0.00012489582877606153, 0.00013577050413005054, 0.00013816128193866462, 0.00010577926877886057, 9.71648478298448e-05, 0.00012717257777694613, 0.00010892276623053476, 9.74036956904456e-05, 9.133153798757121e-05, 0.0001003268116619438, 9.828891052166e-05, 9.07535504666157e-05, 8.539887494407594e-05, 8.375250035896897e-05, 8.515224908478558e-05, 8.622110908618197e-05, 7.238877878990024e-05, 8.275114669231698e-05, 7.396119326585904e-05, 7.161871326388791e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.0001664285200604354, "first_to_last_ratio": 1.9719337364651075, "imbalance_abs_log_ratio": 0.6790146534176558, "last_quartile_mean": 8.439863722742302e-05, "mean": 0.0001245867656507471, "normalized": [ 0.8984334461398115, 1.5927952483854981, 1.5658595485525297, 1.3198735865176374, 1.131213894321221, 1.5125315092450449, 1.3796810696180641, 1.286366052942558, 1.0960606282761096, 1.4536884557404346, 1.1543342746982923, 1.0797559630746076, 1.0497530570909623, 1.1457182987306291, 1.1551962694199982, 0.9023721663305979, 0.8238710779144349, 1.0632503993742801, 0.9211624832492573, 0.8274335806257342, 0.7739032412019898, 0.850990856837025, 0.8281317613098801, 0.7681943949939335, 0.7208251570035091, 0.7089546837555952, 0.7203162530736348, 0.7236174125342815, 0.6145740290254166, 0.6994991973458914, 0.6274381906330746, 0.6042038120380651 ], "peak_layer": 2, "peak_normalized": 1.5927952483854981, "population_cv": 0.2889665774838781, "rest_mean": 0.00012946851159169995, "spike_contrast": 0.7586812913967177, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.822533756960183e-05, "top_five_layers": [ 2, 3, 6, 10, 7 ], "top_five_spike_overlap": 0 }, "values": [ 0.00011193291720701382, 0.0001984412083402276, 0.0001950853766174987, 0.000164438781212084, 0.00014093428035266697, 0.00018844140868168324, 0.00017189000209327787, 0.00016026418597903103, 0.00013655464863404632, 0.00018111034296453, 0.00014381477376446128, 0.0001345233031315729, 0.00013078533811494708, 0.00014274133718572557, 0.00014392216689884663, 0.00011242362961638719, 0.00010264343291055411, 0.0001324669283349067, 0.00011476465442683548, 0.00010308727360097691, 9.641810174798593e-05, 0.00010602219845168293, 0.00010317425767425448, 9.570685506332666e-05, 8.980527491075918e-05, 8.832637104205787e-05, 8.974187221610919e-05, 9.015315299620852e-05, 7.656779052922502e-05, 8.714834257261828e-05, 7.817049481673166e-05, 7.527579873567447e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.14759047608822584, "first_to_last_ratio": 1.9719337947467823, "imbalance_abs_log_ratio": 0.6790146829732503, "last_quartile_mean": 0.07484555337578058, "mean": 0.1104847895912826, "normalized": [ 0.8984334974418968, 1.5927952138130326, 1.565859646322909, 1.3198735575362357, 1.1312139236939245, 1.5125315007911495, 1.3796810264894532, 1.286366010451999, 1.0960606168617568, 1.4536884922319049, 1.1543342013148132, 1.0797558953237962, 1.0497530983161607, 1.1457182562234747, 1.1551962948455465, 0.9023721940039697, 0.8238711061937815, 1.0632504861804335, 0.9211625151546727, 0.8274335809310764, 0.7739032642161865, 0.8509909070635955, 0.8281317391228538, 0.7681943896830757, 0.7208251689669645, 0.7089546589522648, 0.7203162343708721, 0.7236173971219663, 0.6145740294939214, 0.6994991437375198, 0.6274381319743458, 0.6042038211744466 ], "peak_layer": 2, "peak_normalized": 1.5927952138130326, "population_cv": 0.28896658018674404, "rest_mean": 0.11481397047086998, "spike_contrast": 0.7586813040631772, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.08710721284151077, "top_five_layers": [ 2, 3, 6, 10, 7 ], "top_five_spike_overlap": 0 }, "values": [ 0.09926323592662811, 0.1759796440601349, 0.1730036735534668, 0.14582595229148865, 0.12498193234205246, 0.16711172461509705, 0.15243376791477203, 0.14212387800216675, 0.12109802663326263, 0.16061046719551086, 0.1275363713502884, 0.1192966029047966, 0.11598175019025803, 0.12658444046974182, 0.12763161957263947, 0.09969840198755264, 0.0910252258181572, 0.11747300624847412, 0.10177444666624069, 0.09141882508993149, 0.08550453931093216, 0.09402155131101608, 0.09149596095085144, 0.08487379550933838, 0.0796402171254158, 0.0783287063241005, 0.07958398759365082, 0.07994871586561203, 0.06790108233690262, 0.07728401571512222, 0.069322369992733, 0.06675533205270767 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 5.093040158499207e-05, "first_to_last_ratio": 0.8860093039525244, "imbalance_abs_log_ratio": 0.12102782735710801, "last_quartile_mean": 5.748291960117058e-05, "mean": 5.7093861016710434e-05, "normalized": [ 0.4337136137178416, 0.9185569667519601, 0.9444095926066538, 0.8728787927927439, 0.7849140301431445, 1.1296292591692207, 1.0726593944235638, 0.9796135302182263, 0.9386642177229508, 1.3046353648934683, 1.0461832118506482, 1.0032917361864164, 0.9662905330123907, 1.0760080966276753, 1.1828739682986207, 0.9087955578115051, 0.9194941496879511, 1.198292373503509, 1.0946132296674007, 1.0673695485587213, 0.9244260207272295, 1.0525679458300632, 1.125653431968099, 0.9999504891362015, 0.9629027708994021, 1.0063234987710272, 0.9975933861195996, 1.0791525148711678, 0.9412468232151083, 1.085173475571988, 1.0031556955576197, 0.9789667796878816 ], "peak_layer": 10, "peak_normalized": 1.3046353648934683, "population_cv": 0.1441687540611157, "rest_mean": 5.695535414647397e-05, "spike_contrast": 1.0155638391297448, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.784179811598733e-05, "top_five_layers": [ 10, 18, 15, 6, 23 ], "top_five_spike_overlap": 1 }, "values": [ 2.4762384782661684e-05, 5.2443963795667514e-05, 5.3919990023132414e-05, 4.9836020480142906e-05, 4.481377254705876e-05, 6.449489592341706e-05, 6.124226638348773e-05, 5.592991874436848e-05, 5.3591964388033375e-05, 7.448667020071298e-05, 5.973063889541663e-05, 5.728179894504137e-05, 5.516925739357248e-05, 6.143345672171563e-05, 6.75348419463262e-05, 5.1886647270293906e-05, 5.249747118796222e-05, 6.84151382301934e-05, 6.249569560168311e-05, 6.094024865888059e-05, 5.277905074763112e-05, 6.009516800986603e-05, 6.426790059776977e-05, 5.7091034250333905e-05, 5.4975836974335834e-05, 5.74548939766828e-05, 5.695645813830197e-05, 6.161298369988799e-05, 5.373941530706361e-05, 6.19567435933277e-05, 5.727403186028823e-05, 5.58929932594765e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 5.801741053801379e-05, "first_to_last_ratio": 0.954195857595535, "imbalance_abs_log_ratio": 0.046886327142486896, "last_quartile_mean": 6.08024129178375e-05, "mean": 6.296396202287724e-05, "normalized": [ 0.4796364558374551, 0.9568353419130547, 0.9804638447804884, 0.8949545354119591, 0.8237377234766136, 1.1597846916175047, 1.085649359633547, 0.9904450076993521, 0.9595078978373608, 1.2941748206155441, 1.0680753760655664, 1.0231559828609242, 0.9904019047543832, 1.096490963646882, 1.1850351594596897, 0.9422215946487361, 0.9418631353047864, 1.1705024179981143, 1.0926353878857977, 1.0382275689133567, 0.9320833313178482, 1.0784583323883348, 1.0985080196909318, 0.9917906747078287, 0.9476388143731139, 0.9855066582268529, 0.978396290111904, 1.028407839812513, 0.8982055143685478, 1.0282804954549911, 0.9451124964564701, 0.9138123627295472 ], "peak_layer": 10, "peak_normalized": 1.2941748206155441, "population_cv": 0.13431233496292436, "rest_mean": 6.285090866084729e-05, "spike_contrast": 1.0115120295379672, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.357445017783903e-05, "top_five_layers": [ 10, 15, 18, 6, 23 ], "top_five_spike_overlap": 1 }, "values": [ 3.019981159013696e-05, 6.0246144130360335e-05, 6.173388828756288e-05, 5.634988337988034e-05, 5.186579073779285e-05, 7.302463927771896e-05, 6.835678505012766e-05, 6.236234185053036e-05, 6.041441884008236e-05, 8.148637425620109e-05, 6.725025741616264e-05, 6.442195444833487e-05, 6.235962791834027e-05, 6.903941539349034e-05, 7.461450877599418e-05, 5.9326004702597857e-05, 5.930343468207866e-05, 7.369946979451925e-05, 6.879665306769311e-05, 6.537092122016475e-05, 5.8687659475253895e-05, 6.790400948375463e-05, 6.916641723364592e-05, 6.244707037694752e-05, 5.966709431959316e-05, 6.205140380188823e-05, 6.160370685392991e-05, 6.475263216998428e-05, 5.655457789544016e-05, 6.474461406469345e-05, 5.950802733423188e-05, 5.753724690293893e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 1.7888660295284353e-05, "first_to_last_ratio": 0.41855462748958633, "imbalance_abs_log_ratio": 0.8709478659353102, "last_quartile_mean": 4.2739129185065394e-05, "mean": 3.1986895351110434e-05, "normalized": [ 0.2593955501951102, 0.4997829486650695, 0.48480994473028793, 0.4968760363401999, 0.5474125689219601, 0.7788406177150305, 0.7554473604850604, 0.6514322393717875, 0.7578135841604356, 0.9674810188795085, 0.9032772556875033, 0.9019220651946442, 0.8311356963419073, 0.9587970177975769, 1.1401394869948454, 0.8953886492354074, 1.0262272842074007, 1.1413916918814533, 1.174037958462901, 1.194927715786966, 1.0267140632187253, 1.3434898267078448, 1.3729067467361273, 1.2011929483563188, 1.1735959904166378, 1.3266745689353452, 1.2739952988364465, 1.4049535244931144, 1.2752889026856997, 1.4680979746684197, 1.4182292846928506, 1.3483241791974143 ], "peak_layer": 30, "peak_normalized": 1.4680979746684197, "population_cv": 0.3184955307507873, "rest_mean": 3.066251991657299e-05, "spike_contrast": 1.276428773763581, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.913852269761264e-05, "top_five_layers": [ 30, 31, 28, 23, 32 ], "top_five_spike_overlap": 1 }, "values": [ 8.297258318634704e-06, 1.5986504877218977e-05, 1.5507564967265353e-05, 1.589352177688852e-05, 1.7510028555989265e-05, 2.491269333404489e-05, 2.416441566310823e-05, 2.083729486912489e-05, 2.4240103812189773e-05, 3.094671410508454e-05, 2.889303505071439e-05, 2.8849686714238487e-05, 2.6585450541460887e-05, 3.066893987124786e-05, 3.6469522456172854e-05, 2.8640703021665104e-05, 3.282582474639639e-05, 3.6509576602838933e-05, 3.755382931558415e-05, 3.822202779701911e-05, 3.2841395295690745e-05, 4.2974068492185324e-05, 4.391502443468198e-05, 3.842243313556537e-05, 3.7539692129939795e-05, 4.243620060151443e-05, 4.0751154301688075e-05, 4.494010136113502e-05, 4.0792532672639936e-05, 4.695989628089592e-05, 4.5364751713350415e-05, 4.3128704419359565e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00022091567916504573, "first_to_last_ratio": 1.232986382535498, "imbalance_abs_log_ratio": 0.20943917994887448, "last_quartile_mean": 0.00017917122386279516, "mean": 0.0002198539991695725, "normalized": [ 0.4672441326905094, 1.0299890566632133, 1.08986441571188, 1.0084939820045402, 0.8814237539069011, 1.2693826047962031, 1.1886514905905403, 1.1035827467935222, 1.0473307057033197, 1.4274044616309502, 1.1442998336482333, 1.0911283607583675, 1.1094038041384326, 1.1946873974128753, 1.2511466100570947, 0.9923165311956169, 0.9239703707694655, 1.2473578196113966, 1.1181336030529359, 1.0160745414007064, 0.9397394354798955, 0.9905446518023159, 1.0121913654482901, 0.9359943959598221, 0.8839446941554127, 0.8510229499852975, 0.8648513533654478, 0.8786898836618169, 0.7455559493639876, 0.8399702482950041, 0.7392957942269532, 0.7163130557190521 ], "peak_layer": 10, "peak_normalized": 1.4274044616309502, "population_cv": 0.18952709251819305, "rest_mean": 0.0002217885959436427, "spike_contrast": 0.9441746799407325, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00020940717658959329, "top_five_layers": [ 10, 6, 15, 18, 14 ], "top_five_spike_overlap": 0 }, "values": [ 0.00010272549116052687, 0.00022644721320830286, 0.00023961105034686625, 0.00022172143508214504, 0.0001937845372594893, 0.0002790788421407342, 0.0002613297838252038, 0.00024262708029709756, 0.00023025984410196543, 0.000313820579322055, 0.00025157889467664063, 0.00023988893372006714, 0.00024390686303377151, 0.000262656802078709, 0.00027506958576850593, 0.0002181647578254342, 0.00020313858112785965, 0.00027423660503700376, 0.0002458261442370713, 0.00022338805138133466, 0.00020660547306761146, 0.00021777520305477083, 0.00022253431961871684, 0.00020578211115207523, 0.0001943387760547921, 0.00018710079893935472, 0.0001901410287246108, 0.00019318348495289683, 0.00016391345707233995, 0.00018467081827111542, 0.00016253713693004102, 0.00015748428995721042 ] } } }, "pre_mlp_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "2999a257265024350fd2665e2a8d452a03cd662193b4245d15290408ad29922d", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.014061377383768559, 0.014165290631353855, 0.014281988143920898, 0.014350161887705326, 0.00966146495193243, 0.009741888381540775, 0.009820535778999329, 0.00991273857653141, 0.0074927168898284435, 0.007546746172010899, 0.00766373984515667, 0.007762390188872814, 0.006325984839349985, 0.006375366821885109, 0.006445344537496567, 0.006555322092026472, 0.005531130358576775, 0.005611324217170477, 0.0056777093559503555, 0.00574630219489336, 0.004946626722812653, 0.0050392793491482735, 0.005084683652967215, 0.005203457549214363, 0.004667261149734259, 0.004717131145298481, 0.004768412560224533, 0.004826710093766451, 0.0044052014127373695, 0.004504534415900707, 0.004574235528707504, 0.004644096828997135 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 3.729895206561196e-05, "first_to_last_ratio": 1.2469721805352876, "imbalance_abs_log_ratio": 0.22071835733540474, "last_quartile_mean": 2.9911615229138988e-05, "mean": 3.340667609563752e-05, "normalized": [ 1.308752778077389, 1.2456464596610868, 1.0236956931411572, 0.9922285647386262, 1.26903181062375, 1.1170846373362797, 1.0287316533628064, 0.9469237624012169, 1.1300261751959062, 1.1132698789554223, 1.0126240729566007, 0.9379202574439188, 1.1130503370514597, 1.070713488608195, 0.9897642722064988, 0.922085688720865, 1.036831181462568, 0.9898105546019274, 0.9395130252451869, 0.9001967292772246, 0.9975952535130206, 0.9404117749145336, 0.9224862220158919, 0.8885787590271026, 0.9461670180110204, 0.8974372095121397, 0.8904691498559152, 0.8684437934704528, 0.9244147277585869, 0.9015689206268677, 0.8740901912628669, 0.8604359589635168 ], "peak_layer": 1, "peak_normalized": 1.308752778077389, "population_cv": 0.1164905978879517, "rest_mean": 3.378375206213807e-05, "spike_contrast": 0.9285666618330386, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.137046587653458e-05, "top_five_layers": [ 1, 5, 2, 9, 6 ], "top_five_spike_overlap": 0 }, "values": [ 4.3721080146497115e-05, 4.161290780757554e-05, 3.419827044126578e-05, 3.314705827506259e-05, 4.239413465256803e-05, 3.73180846509058e-05, 3.4366505133220926e-05, 3.1633575417799875e-05, 3.7750418414361775e-05, 3.719064625329338e-05, 3.3828404411906376e-05, 3.1332798243965954e-05, 3.718331208801828e-05, 3.5768978705164045e-05, 3.306473445263691e-05, 3.080381793552078e-05, 3.463708344497718e-05, 3.3066280593629926e-05, 3.138600732199848e-05, 3.007258055731654e-05, 3.332634150865488e-05, 3.1416031561093405e-05, 3.0817198421573266e-05, 2.9684462788281962e-05, 3.1608295103069395e-05, 2.9980394174344838e-05, 2.9747614462394267e-05, 2.9011820515734144e-05, 3.0881623388268054e-05, 3.01184209092753e-05, 2.9200447897892445e-05, 2.874430538213346e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 6.705871965095866e-05, "first_to_last_ratio": 0.7180297289018083, "imbalance_abs_log_ratio": 0.33124430563536084, "last_quartile_mean": 9.33926785364747e-05, "mean": 8.228966771639534e-05, "normalized": [ 0.8685826408392191, 0.8249006353131848, 0.7154905339803773, 0.6884730939760333, 0.9470487095509537, 0.8638069620260335, 0.8256155016985178, 0.7853664463613442, 0.9851686390305121, 0.9692511243547932, 0.9083804947955202, 0.8729855457896263, 1.0422548534804916, 1.0255492616133721, 0.9871528463964067, 0.9398223255303129, 1.0958375595009802, 1.0726325633807163, 1.0495259774381736, 1.0230838788771643, 1.1576441025653552, 1.1142532603054807, 1.1008567438213033, 1.056908774400763, 1.159063578749664, 1.1192434432436902, 1.1032241584740294, 1.1060435702772593, 1.1786235078649367, 1.1628365878275773, 1.141840825041133, 1.1085318534950752 ], "peak_layer": 29, "peak_normalized": 1.1786235078649367, "population_cv": 0.1341844256762087, "rest_mean": 8.049536754023629e-05, "spike_contrast": 1.1426606459269575, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.197888866765424e-05, "top_five_layers": [ 29, 30, 25, 21, 31 ], "top_five_spike_overlap": 2 }, "values": [ 7.14753768988885e-05, 6.788079917896539e-05, 5.887747829547152e-05, 5.66542221349664e-05, 7.793232362018898e-05, 7.108238787623122e-05, 6.793962529627606e-05, 6.462754390668124e-05, 8.106919995043427e-05, 7.975935295689851e-05, 7.475032907677814e-05, 7.183769048424438e-05, 8.576680556870997e-05, 8.4392107964959e-05, 8.123247971525416e-05, 7.733766688033938e-05, 9.017610864248127e-05, 8.826657722238451e-05, 8.636514394311234e-05, 8.418923243880272e-05, 9.526214853394777e-05, 9.169153054244816e-05, 9.0589135652408e-05, 8.697267185198143e-05, 9.537895675748587e-05, 9.210217103827745e-05, 9.078394941752777e-05, 9.101595787797123e-05, 9.698853682493791e-05, 9.56894364207983e-05, 9.396170207764953e-05, 9.122071787714958e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.05946834245696664, "first_to_last_ratio": 0.7180297222365345, "imbalance_abs_log_ratio": 0.33124431491808765, "last_quartile_mean": 0.08282156102359295, "mean": 0.07297530013602227, "normalized": [ 0.8685826122359729, 0.8249005917343738, 0.71549052911611, 0.6884731350635278, 0.9470486699630107, 0.8638070113577097, 0.8256154769796886, 0.7853664269052657, 0.9851686341386658, 0.9692511556795396, 0.9083805454214465, 0.8729855579949275, 1.0422548040385637, 1.0255492365750694, 0.9871528950273681, 0.9398223256645052, 1.0958375053304956, 1.07263252629438, 1.0495259690486791, 1.0230837911446407, 1.1576441435704576, 1.1142533045483842, 1.1008568124470575, 1.0569088284972943, 1.1590636022344982, 1.119243411839159, 1.1032241423789504, 1.1060435591240758, 1.1786235037173105, 1.1628366076958818, 1.141840809884862, 1.1085318743481278 ], "peak_layer": 29, "peak_normalized": 1.1786235037173105, "population_cv": 0.13418442668798974, "rest_mean": 0.07138409617322462, "spike_contrast": 1.1426607032635474, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.08156780153512955, "top_five_layers": [ 29, 30, 25, 21, 31 ], "top_five_spike_overlap": 2 }, "values": [ 0.06338507682085037, 0.0601973682641983, 0.05221313610672951, 0.050241533666849136, 0.0691111609339714, 0.06303657591342926, 0.060249537229537964, 0.05731235072016716, 0.07189297676086426, 0.07073139399290085, 0.06628934293985367, 0.06370638310909271, 0.07605885714292526, 0.0748397633433342, 0.07203777879476547, 0.06858381628990173, 0.07996907085180283, 0.07827568054199219, 0.07658947259187698, 0.07465984672307968, 0.0844794288277626, 0.08131296932697296, 0.0803353562951088, 0.07712823897600174, 0.08458301424980164, 0.08167712390422821, 0.08050811290740967, 0.08071386069059372, 0.0860104039311409, 0.08485835045576096, 0.08332617580890656, 0.08089544624090195 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 7.408341275549901e-06, "first_to_last_ratio": 0.49124437919492936, "imbalance_abs_log_ratio": 0.7108135577065074, "last_quartile_mean": 1.5080765479069669e-05, "mean": 1.1844782648040564e-05, "normalized": [ 0.5657990507718705, 0.5382233871697658, 0.49942145823009426, 0.48906984415931193, 0.8062921756393574, 0.7122953827323945, 0.7013508394120195, 0.6911626993599347, 0.8940675881372303, 0.868722127857376, 0.9735911331661544, 0.854855860776803, 0.989483587379368, 1.0059088486455374, 1.0303729006301217, 0.9941025538400318, 1.1620912657632405, 1.127012997009919, 1.113401270263767, 1.066097770938846, 1.1826034542812638, 1.1643547935214302, 1.2049778184223279, 1.1791491538213539, 1.2365172470752577, 1.219921447624247, 1.2337761202582145, 1.2288278254433027, 1.3039217543779833, 1.3631114810840155, 1.2856635723507386, 1.3138525898567213 ], "peak_layer": 30, "peak_normalized": 1.3631114810840155, "population_cv": 0.2549101540724733, "rest_mean": 1.1420299651271543e-05, "spike_contrast": 1.237882653019464, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.4136990830593277e-05, "top_five_layers": [ 30, 32, 29, 31, 25 ], "top_five_spike_overlap": 1 }, "values": [ 6.701766778860474e-06, 6.3751390371180605e-06, 5.915538622502936e-06, 5.7929260037781205e-06, 9.550355571263935e-06, 8.436983989668079e-06, 8.307348252856173e-06, 8.186671948351432e-06, 1.0590036254143342e-05, 1.0289824786013924e-05, 1.1531975360412616e-05, 1.0125581866304856e-05, 1.1720218026312068e-05, 1.1914771675947122e-05, 1.220454305439489e-05, 1.1774928680097219e-05, 1.3764718460151926e-05, 1.334922399109928e-05, 1.3187996046326589e-05, 1.2627696378331166e-05, 1.4007680874783546e-05, 1.379152945446549e-05, 1.4272700354922563e-05, 1.3966765436634887e-05, 1.4646278032159898e-05, 1.4449704394792207e-05, 1.4613809980801307e-05, 1.4555198504240252e-05, 1.5444669770658948e-05, 1.614575921848882e-05, 1.5228405572997872e-05, 1.5562298358418047e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 5.6029601637419546e-05, "first_to_last_ratio": 0.6070213107493427, "imbalance_abs_log_ratio": 0.49919138022039955, "last_quartile_mean": 9.230252817360451e-05, "mean": 7.840498903988191e-05, "normalized": [ 0.6598133090619259, 0.6419677920829977, 0.6005256373772865, 0.5913463573822231, 0.8562376513989928, 0.8062968559250673, 0.7923156566726156, 0.768439227921366, 0.9790122729268586, 0.9665730330981341, 0.9163710065350301, 0.8881686280668529, 1.0631190324760253, 1.0511030519715439, 1.0188080221730096, 0.9712812246823179, 1.1361272291219966, 1.1138561419734825, 1.0918134348286874, 1.0639217496772315, 1.203414696914147, 1.1592426077267617, 1.1442664103435203, 1.097952968622282, 1.2042281788778202, 1.1625445130141736, 1.1452986211445626, 1.1473932954411665, 1.2233942816523278, 1.205318760675343, 1.1829469858778916, 1.146901364356358 ], "peak_layer": 29, "peak_normalized": 1.2233942816523278, "population_cv": 0.1898597249088972, "rest_mean": 7.605543874820728e-05, "spike_contrast": 1.1977126437531995, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.109256061492487e-05, "top_five_layers": [ 29, 30, 25, 21, 31 ], "top_five_spike_overlap": 2 }, "values": [ 5.173265526536852e-05, 5.033347770222463e-05, 4.708420601673424e-05, 4.636450466932729e-05, 6.713330367347226e-05, 6.321769615169615e-05, 6.212150037754327e-05, 6.024946924299002e-05, 7.675944652874023e-05, 7.578414806630462e-05, 7.184805872384459e-05, 6.963685154914856e-05, 8.335383608937263e-05, 8.241172326961532e-05, 7.987963181221858e-05, 7.615329377586022e-05, 8.907804294722155e-05, 8.733187860343605e-05, 8.560362039133906e-05, 8.341677312273532e-05, 9.435371612198651e-05, 9.089040395338088e-05, 8.971619536168873e-05, 8.608499047113582e-05, 9.441749716643244e-05, 9.114928980125114e-05, 8.97971258382313e-05, 8.996135875349864e-05, 9.592021524440497e-05, 9.450300422031432e-05, 9.274894546251744e-05, 8.992278890218586e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 4.8511469685763586e-05, "first_to_last_ratio": 0.5271352332510719, "imbalance_abs_log_ratio": 0.6402981537748061, "last_quartile_mean": 9.202850924339145e-05, "mean": 7.532146378252946e-05, "normalized": [ 0.548104082267993, 0.5416830199005969, 0.528884702689083, 0.5247448675895723, 0.7777813723908718, 0.7487613756360456, 0.7496059866168319, 0.7329069161583202, 0.9480676391902623, 0.9399123881532756, 0.9000936195631334, 0.883441930780847, 1.0560589316963704, 1.0534552095740957, 1.032095781768693, 0.9835620691098637, 1.1601754560133901, 1.1467788545782247, 1.1233340534265408, 1.0967375255670042, 1.2396083970871608, 1.1951467072205837, 1.1828066060767042, 1.1317734013234224, 1.244698956870482, 1.2025862582796845, 1.1881970083126314, 1.1882256015362047, 1.2689460104605979, 1.2589162611011824, 1.2307315535498626, 1.1921774555104672 ], "peak_layer": 29, "peak_normalized": 1.2689460104605979, "population_cv": 0.23486334953575583, "rest_mean": 7.254842300123225e-05, "spike_contrast": 1.244629176846487, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.029588400153444e-05, "top_five_layers": [ 29, 30, 25, 21, 31 ], "top_five_spike_overlap": 2 }, "values": [ 4.1284001781605184e-05, 4.080035796505399e-05, 3.9836369978729635e-05, 3.952455153921619e-05, 5.858363147126511e-05, 5.639780283672735e-05, 5.6461420172126964e-05, 5.520362174138427e-05, 7.140984234865755e-05, 7.079557690303773e-05, 6.77963689668104e-05, 6.654213939327747e-05, 7.954390457598493e-05, 7.934778841445222e-05, 7.773896504659206e-05, 7.408333476632833e-05, 8.738611359149218e-05, 8.637706196168438e-05, 8.461116522084922e-05, 8.26078758109361e-05, 9.336911898571998e-05, 9.002019942272455e-05, 8.909072494134307e-05, 8.524682925781235e-05, 9.375254740007222e-05, 9.058055729838088e-05, 8.949673792812973e-05, 8.949889161158353e-05, 9.557887096889317e-05, 9.482341556577012e-05, 9.270050213672221e-05, 8.979655103757977e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 9.61669284151867e-05, "first_to_last_ratio": 0.8297833624837898, "imbalance_abs_log_ratio": 0.18659062131596024, "last_quartile_mean": 0.00011589401856326731, "mean": 0.00010716335873439675, "normalized": [ 0.9380742209455786, 0.8898820858744699, 0.7837417114644826, 0.7583102664636657, 1.0822070619439073, 0.9520749081713696, 0.9107982467806989, 0.8640017807748708, 1.0669538303069324, 1.0266950649381863, 0.9594029855154438, 0.8901506143407661, 1.0795289063552718, 1.0379165674077444, 0.9706971366471856, 0.914979822300863, 1.052939971267624, 1.0225707935964177, 0.9851238030379684, 0.9636652893153618, 1.0990044510799908, 1.0523904893769196, 1.0422234788441622, 1.0049019599938585, 1.0962247910521359, 1.0599378706264646, 1.05039930496957, 1.0583562345651827, 1.1199788667078585, 1.1100529546792268, 1.0850973923636869, 1.0717171382921347 ], "peak_layer": 29, "peak_normalized": 1.1199788667078585, "population_cv": 0.09156717959150794, "rest_mean": 0.00010599351123716958, "spike_contrast": 1.0706366257223145, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00011348053521942347, "top_five_layers": [ 29, 30, 21, 25, 31 ], "top_five_spike_overlap": 2 }, "values": [ 0.00010052718425868079, 9.536275319987908e-05, 8.398839418077841e-05, 8.12630751170218e-05, 0.00011597294360399246, 0.00010202754492638633, 9.760419925441965e-05, 9.25893327803351e-05, 0.00011433835607022047, 0.00011002409155480564, 0.00010281284630764276, 9.539152961224318e-05, 0.000115685943455901, 0.0001112266254494898, 0.0001040231654769741, 9.805231093196198e-05, 0.00011283658386673778, 0.00010958212078548968, 0.00010556917550275102, 0.00010326960909878835, 0.00011777300824178383, 0.0001127776995417662, 0.00011168816854478791, 0.00010768866923172027, 0.00011747513053705916, 0.00011358650226611644, 0.00011256431753281504, 0.00011341700883349404, 0.00012002069706795737, 0.00011895700299646705, 0.00011628268111962825, 0.00011484880815260112 ] } } } }, "selector": { "expected_uniform_count": 16, "identities_unique": true, "identity_order_sha256": "97fd9721a263024f6a746638f32e0ad9a988cd4c111c00e9e178c1022e5c9d5e", "passed": true, "uniform_count": 16, "uniform_indices": [ 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55 ], "visit_count": 65, "visits": [ { "branch": "attention", "coefficient": "detached_learned", "group": 1, "index": 0, "kind": "depth", "layer": 1, "offset": 1, "sources": 1, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 1, "index": 1, "kind": "depth", "layer": 1, "offset": 1, "sources": 2, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 1, "index": 2, "kind": "depth", "layer": 2, "offset": 2, "sources": 2, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 1, "index": 3, "kind": "depth", "layer": 2, "offset": 2, "sources": 2, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 1, "index": 4, "kind": "depth", "layer": 3, "offset": 3, "sources": 2, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 1, "index": 5, "kind": "depth", "layer": 3, "offset": 3, "sources": 2, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 1, "index": 6, "kind": "depth", "layer": 4, "offset": 4, "sources": 2, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 1, "index": 7, "kind": "depth", "layer": 4, "offset": 4, "sources": 2, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 2, "index": 8, "kind": "depth", "layer": 5, "offset": 1, "sources": 2, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 2, "index": 9, "kind": "depth", "layer": 5, "offset": 1, "sources": 3, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 2, "index": 10, "kind": "depth", "layer": 6, "offset": 2, "sources": 3, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 2, "index": 11, "kind": "depth", "layer": 6, "offset": 2, "sources": 3, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 2, "index": 12, "kind": "depth", "layer": 7, "offset": 3, "sources": 3, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 2, "index": 13, "kind": "depth", "layer": 7, "offset": 3, "sources": 3, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 2, "index": 14, "kind": "depth", "layer": 8, "offset": 4, "sources": 3, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 2, "index": 15, "kind": "depth", "layer": 8, "offset": 4, "sources": 3, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 3, "index": 16, "kind": "depth", "layer": 9, "offset": 1, "sources": 3, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 3, "index": 17, "kind": "depth", "layer": 9, "offset": 1, "sources": 4, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 3, "index": 18, "kind": "depth", "layer": 10, "offset": 2, "sources": 4, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 3, "index": 19, "kind": "depth", "layer": 10, "offset": 2, "sources": 4, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 3, "index": 20, "kind": "depth", "layer": 11, "offset": 3, "sources": 4, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 3, "index": 21, "kind": "depth", "layer": 11, "offset": 3, "sources": 4, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 3, "index": 22, "kind": "depth", "layer": 12, "offset": 4, "sources": 4, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 3, "index": 23, "kind": "depth", "layer": 12, "offset": 4, "sources": 4, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 4, "index": 24, "kind": "depth", "layer": 13, "offset": 1, "sources": 4, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 4, "index": 25, "kind": "depth", "layer": 13, "offset": 1, "sources": 5, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 4, "index": 26, "kind": "depth", "layer": 14, "offset": 2, "sources": 5, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 4, "index": 27, "kind": "depth", "layer": 14, "offset": 2, "sources": 5, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 4, "index": 28, "kind": "depth", "layer": 15, "offset": 3, "sources": 5, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 4, "index": 29, "kind": "depth", "layer": 15, "offset": 3, "sources": 5, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 4, "index": 30, "kind": "depth", "layer": 16, "offset": 4, "sources": 5, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 4, "index": 31, "kind": "depth", "layer": 16, "offset": 4, "sources": 5, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 5, "index": 32, "kind": "depth", "layer": 17, "offset": 1, "sources": 5, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 5, "index": 33, "kind": "depth", "layer": 17, "offset": 1, "sources": 6, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 5, "index": 34, "kind": "depth", "layer": 18, "offset": 2, "sources": 6, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 5, "index": 35, "kind": "depth", "layer": 18, "offset": 2, "sources": 6, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 5, "index": 36, "kind": "depth", "layer": 19, "offset": 3, "sources": 6, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 5, "index": 37, "kind": "depth", "layer": 19, "offset": 3, "sources": 6, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 5, "index": 38, "kind": "depth", "layer": 20, "offset": 4, "sources": 6, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 5, "index": 39, "kind": "depth", "layer": 20, "offset": 4, "sources": 6, "use_uniform": false }, { "branch": "attention", "coefficient": "uniform", "group": 6, "index": 40, "kind": "depth", "layer": 21, "offset": 1, "sources": 6, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 6, "index": 41, "kind": "depth", "layer": 21, "offset": 1, "sources": 7, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 6, "index": 42, "kind": "depth", "layer": 22, "offset": 2, "sources": 7, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 6, "index": 43, "kind": "depth", "layer": 22, "offset": 2, "sources": 7, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 6, "index": 44, "kind": "depth", "layer": 23, "offset": 3, "sources": 7, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 6, "index": 45, "kind": "depth", "layer": 23, "offset": 3, "sources": 7, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 6, "index": 46, "kind": "depth", "layer": 24, "offset": 4, "sources": 7, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 6, "index": 47, "kind": "depth", "layer": 24, "offset": 4, "sources": 7, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 7, "index": 48, "kind": "depth", "layer": 25, "offset": 1, "sources": 7, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 7, "index": 49, "kind": "depth", "layer": 25, "offset": 1, "sources": 8, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 7, "index": 50, "kind": "depth", "layer": 26, "offset": 2, "sources": 8, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 7, "index": 51, "kind": "depth", "layer": 26, "offset": 2, "sources": 8, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 7, "index": 52, "kind": "depth", "layer": 27, "offset": 3, "sources": 8, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 7, "index": 53, "kind": "depth", "layer": 27, "offset": 3, "sources": 8, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 7, "index": 54, "kind": "depth", "layer": 28, "offset": 4, "sources": 8, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 7, "index": 55, "kind": "depth", "layer": 28, "offset": 4, "sources": 8, "use_uniform": true }, { "branch": "attention", "coefficient": "detached_learned", "group": 8, "index": 56, "kind": "depth", "layer": 29, "offset": 1, "sources": 8, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 8, "index": 57, "kind": "depth", "layer": 29, "offset": 1, "sources": 9, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 8, "index": 58, "kind": "depth", "layer": 30, "offset": 2, "sources": 9, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 8, "index": 59, "kind": "depth", "layer": 30, "offset": 2, "sources": 9, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 8, "index": 60, "kind": "depth", "layer": 31, "offset": 3, "sources": 9, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 8, "index": 61, "kind": "depth", "layer": 31, "offset": 3, "sources": 9, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 8, "index": 62, "kind": "depth", "layer": 32, "offset": 4, "sources": 9, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 8, "index": 63, "kind": "depth", "layer": 32, "offset": 4, "sources": 9, "use_uniform": false }, { "branch": "output", "coefficient": "detached_learned", "group": null, "index": 64, "kind": "output", "layer": null, "offset": null, "sources": 9, "use_uniform": false } ], "visits_sha256": "ecae65dd230eef0be68361d7f87960e32badc2e7c90cab749f34e981d307b2d6" } }, "uniform_output_only": { "bits_per_byte": 7.959341947457508, "forward": { "activation_sha256": { "attention_branch_output": "676759a2d7c0615ad7e1f246e90927fb829d00c8b6c3745032ae6b27ab7d902a", "mlp_branch_output": "afe29f3ef4d2992353f0bb59fe7c5c2c5050fca671a53184825a8d63cd83a2bc", "post_attention_state": "31853ffb07b24cc59e774bb20656840b4358b3080b42516d380f2dc0f46404c8", "post_mlp_state": "7fd9c911e93175896ad618842c120e77b3f017c17b7eec9c2bfb01fc6dbc8ac2", "pre_attention_input": "de71d4cf2d509a1063f6eac7f50c4885e9a058dee7e4b993ad40f5527f528d00", "pre_mlp_input": "2999a257265024350fd2665e2a8d452a03cd662193b4245d15290408ad29922d" }, "logits_sha256": "873a309a197bfd08fa523edcb440d2697903a34abc02dad1c7f5991509c3c151", "loss_nats": 5.516995429992676, "mixer_summary_sha256": "8450b24bdc777885a3357dc7b8080e7410acbe96943686433da76f379295133a" }, "loss_nats": 5.516995429992676, "loss_scale": 1.0, "mixers": [ { "branch": "attention", "entropy_mean": -0.0, "group": 1, "latest_source_mass": 1.0, "layer": 1, "max_source_mass": 1.0, "mixer_index": 0, "normalized_entropy": 1.0, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 1.0, "median": 1.0, "p05": 1.0, "p95": 1.0 } ], "sources": 1 }, { "branch": "mlp", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 1, "max_source_mass": 0.5, "mixer_index": 1, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 2, "max_source_mass": 0.5, "mixer_index": 2, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 2, "max_source_mass": 0.5, "mixer_index": 3, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 3, "max_source_mass": 0.5, "mixer_index": 4, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 3, "max_source_mass": 0.5, "mixer_index": 5, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 4, "max_source_mass": 0.5, "mixer_index": 6, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 4, "max_source_mass": 0.5, "mixer_index": 7, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6931471824645996, "group": 2, "latest_source_mass": 0.5, "layer": 5, "max_source_mass": 0.5, "mixer_index": 8, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "completed_group_1", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 5, "max_source_mass": 0.3333333432674408, "mixer_index": 9, "normalized_entropy": 1.0000000180537645, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 6, "max_source_mass": 0.3333333432674408, "mixer_index": 10, "normalized_entropy": 1.0000000180537645, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 6, "max_source_mass": 0.3333333432674408, "mixer_index": 11, "normalized_entropy": 1.0000000180537645, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 7, "max_source_mass": 0.3333333432674408, "mixer_index": 12, "normalized_entropy": 1.0000000180537645, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 7, "max_source_mass": 0.3333333432674408, "mixer_index": 13, "normalized_entropy": 1.0000000180537645, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 8, "max_source_mass": 0.3333333432674408, "mixer_index": 14, "normalized_entropy": 1.0000000180537645, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 8, "max_source_mass": 0.3333333432674408, "mixer_index": 15, "normalized_entropy": 1.0000000180537645, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0986123085021973, "group": 3, "latest_source_mass": 0.3333333432674408, "layer": 9, "max_source_mass": 0.3333333432674408, "mixer_index": 16, "normalized_entropy": 1.0000000180537645, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_2", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 9, "max_source_mass": 0.25, "mixer_index": 17, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 10, "max_source_mass": 0.25, "mixer_index": 18, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 10, "max_source_mass": 0.25, "mixer_index": 19, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 11, "max_source_mass": 0.25, "mixer_index": 20, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 11, "max_source_mass": 0.25, "mixer_index": 21, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 12, "max_source_mass": 0.25, "mixer_index": 22, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 12, "max_source_mass": 0.25, "mixer_index": 23, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3862943649291992, "group": 4, "latest_source_mass": 0.25, "layer": 13, "max_source_mass": 0.25, "mixer_index": 24, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_3", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 13, "max_source_mass": 0.20000000298023224, "mixer_index": 25, "normalized_entropy": 1.0000000186840277, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 14, "max_source_mass": 0.20000000298023224, "mixer_index": 26, "normalized_entropy": 1.0000000186840277, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 14, "max_source_mass": 0.20000000298023224, "mixer_index": 27, "normalized_entropy": 1.0000000186840277, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 15, "max_source_mass": 0.20000000298023224, "mixer_index": 28, "normalized_entropy": 1.0000000186840277, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 15, "max_source_mass": 0.20000000298023224, "mixer_index": 29, "normalized_entropy": 1.0000000186840277, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 16, "max_source_mass": 0.20000000298023224, "mixer_index": 30, "normalized_entropy": 1.0000000186840277, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 16, "max_source_mass": 0.20000000298023224, "mixer_index": 31, "normalized_entropy": 1.0000000186840277, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.6094379425048828, "group": 5, "latest_source_mass": 0.20000000298023224, "layer": 17, "max_source_mass": 0.20000000298023224, "mixer_index": 32, "normalized_entropy": 1.0000000186840277, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_4", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 17, "max_source_mass": 0.1666666716337204, "mixer_index": 33, "normalized_entropy": 1.0000001451965654, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 18, "max_source_mass": 0.1666666716337204, "mixer_index": 34, "normalized_entropy": 1.0000001451965654, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 18, "max_source_mass": 0.1666666716337204, "mixer_index": 35, "normalized_entropy": 1.0000001451965654, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 19, "max_source_mass": 0.1666666716337204, "mixer_index": 36, "normalized_entropy": 1.0000001451965654, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 19, "max_source_mass": 0.1666666716337204, "mixer_index": 37, "normalized_entropy": 1.0000001451965654, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 20, "max_source_mass": 0.1666666716337204, "mixer_index": 38, "normalized_entropy": 1.0000001451965654, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 20, "max_source_mass": 0.1666666716337204, "mixer_index": 39, "normalized_entropy": 1.0000001451965654, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.791759729385376, "group": 6, "latest_source_mass": 0.1666666716337204, "layer": 21, "max_source_mass": 0.1666666716337204, "mixer_index": 40, "normalized_entropy": 1.0000001451965654, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_5", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 21, "max_source_mass": 0.1428571492433548, "mixer_index": 41, "normalized_entropy": 1.00000003408302, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 22, "max_source_mass": 0.1428571492433548, "mixer_index": 42, "normalized_entropy": 1.00000003408302, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 22, "max_source_mass": 0.1428571492433548, "mixer_index": 43, "normalized_entropy": 1.00000003408302, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 23, "max_source_mass": 0.1428571492433548, "mixer_index": 44, "normalized_entropy": 1.00000003408302, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 23, "max_source_mass": 0.1428571492433548, "mixer_index": 45, "normalized_entropy": 1.00000003408302, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 24, "max_source_mass": 0.1428571492433548, "mixer_index": 46, "normalized_entropy": 1.00000003408302, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 24, "max_source_mass": 0.1428571492433548, "mixer_index": 47, "normalized_entropy": 1.00000003408302, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9459102153778076, "group": 7, "latest_source_mass": 0.1428571492433548, "layer": 25, "max_source_mass": 0.1428571492433548, "mixer_index": 48, "normalized_entropy": 1.00000003408302, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_6", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 25, "max_source_mass": 0.125, "mixer_index": 49, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 26, "max_source_mass": 0.125, "mixer_index": 50, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 26, "max_source_mass": 0.125, "mixer_index": 51, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 27, "max_source_mass": 0.125, "mixer_index": 52, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 27, "max_source_mass": 0.125, "mixer_index": 53, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 28, "max_source_mass": 0.125, "mixer_index": 54, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 28, "max_source_mass": 0.125, "mixer_index": 55, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.079441547393799, "group": 8, "latest_source_mass": 0.125, "layer": 29, "max_source_mass": 0.125, "mixer_index": 56, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_7", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 29, "max_source_mass": 0.1111111119389534, "mixer_index": 57, "normalized_entropy": 1.0000000180537645, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 30, "max_source_mass": 0.1111111119389534, "mixer_index": 58, "normalized_entropy": 1.0000000180537645, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 30, "max_source_mass": 0.1111111119389534, "mixer_index": 59, "normalized_entropy": 1.0000000180537645, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 31, "max_source_mass": 0.1111111119389534, "mixer_index": 60, "normalized_entropy": 1.0000000180537645, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 31, "max_source_mass": 0.1111111119389534, "mixer_index": 61, "normalized_entropy": 1.0000000180537645, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 32, "max_source_mass": 0.1111111119389534, "mixer_index": 62, "normalized_entropy": 1.0000000180537645, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 32, "max_source_mass": 0.1111111119389534, "mixer_index": 63, "normalized_entropy": 1.0000000180537645, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "output", "entropy_mean": 2.1972246170043945, "group": null, "latest_source_mass": 0.1111111119389534, "layer": null, "max_source_mass": 0.1111111119389534, "mixer_index": null, "normalized_entropy": 1.0000000180537645, "offset": null, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_8", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 } ], "mode": "uniform_output_only", "optimizer_state_after": "b82a349700eda58e2330a22ca2ee1f7fb1406efd47fd9aa306122b368a114192", "optimizer_state_before": "b82a349700eda58e2330a22ca2ee1f7fb1406efd47fd9aa306122b368a114192", "optimizer_state_unchanged": true, "positions": { "attention_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "676759a2d7c0615ad7e1f246e90927fb829d00c8b6c3745032ae6b27ab7d902a", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.0022265110164880753, 0.0021613112185150385, 0.0025601957459002733, 0.0025323182344436646, 0.0027253651060163975, 0.002904455177485943, 0.0029208383057266474, 0.00299624796025455, 0.0030937574338167906, 0.0033804215490818024, 0.0034370317589491606, 0.0038969130255281925, 0.004255065228790045, 0.004119740333408117, 0.003970506135374308, 0.004385598469525576, 0.0041999719105660915, 0.005232699681073427, 0.004664790350943804, 0.004898969549685717, 0.005131533369421959, 0.005069608334451914, 0.0051318020559847355, 0.0058372304774820805, 0.0059690922498703, 0.00554921617731452, 0.0058550527319312096, 0.006255472078919411, 0.006502739619463682, 0.006122888531535864, 0.007071305997669697, 0.006241494789719582 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.00015962477027642308, "first_to_last_ratio": 3.7699040072151497, "imbalance_abs_log_ratio": 1.3270495388586014, "last_quartile_mean": 4.234186599205714e-05, "mean": 8.617327489446325e-05, "normalized": [ 2.5428430303860874, 2.3222064178412967, 2.1011513502014654, 1.8911470095267005, 1.7004447966680725, 1.5353700031893112, 1.4229335846259188, 1.3028620492588128, 1.2049362956920768, 1.1705197150696323, 1.0773612761873523, 1.0133200843661747, 0.9906112963967714, 0.9272530070793641, 0.8550207003106389, 0.8185600560490035, 0.7703816559177782, 0.7173918665339755, 0.6760157258502337, 0.6514535263006792, 0.6184138115448249, 0.6053606047045522, 0.5838821473278404, 0.5697017460729616, 0.5457761369570343, 0.5276086337508958, 0.5104655265174319, 0.4951752820428687, 0.47527333481689216, 0.465789239220051, 0.45896414036514044, 0.45180594922816086 ], "peak_layer": 1, "peak_normalized": 2.5428430303860874, "population_cv": 0.5705341151299013, "rest_mean": 9.280180475562466e-05, "spike_contrast": 0.5428688997682254, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.0379213644191625e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00021912511147093028, 0.00020011213200632483, 0.00018106309289578348, 0.00016296633111778647, 0.00014653289690613747, 0.00013230786134954542, 0.0001226188469445333, 0.00011227188952034339, 0.0001038333066389896, 0.00010086751717608422, 9.283974941354245e-05, 8.732111018616706e-05, 8.536421955795959e-05, 7.990442827576771e-05, 7.367993384832516e-05, 7.053800072753802e-05, 6.63863102090545e-05, 6.182000652188435e-05, 5.825448897667229e-05, 5.6137883802875876e-05, 5.329074338078499e-05, 5.216590579948388e-05, 5.031503678765148e-05, 4.909306517220102e-05, 4.703131708083674e-05, 4.5465763832908124e-05, 4.3988486140733585e-05, 4.2670875700423494e-05, 4.095585973118432e-05, 4.013858415419236e-05, 3.955044303438626e-05, 3.893359826179221e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.00021442661636683624, "first_to_last_ratio": 1.843144421770586, "imbalance_abs_log_ratio": 0.6114730379455638, "last_quartile_mean": 0.0001163373926829081, "mean": 0.00015221274247778638, "normalized": [ 1.7741446897870732, 1.648461275666186, 1.5301464224684427, 1.4217121808823485, 1.3265065496562456, 1.2454420544341354, 1.1897998831173078, 1.1336248235930768, 1.0869089599614004, 1.0643003125599004, 1.0227030055526172, 0.9941765648180964, 0.9788958472869692, 0.9511102301475627, 0.9228060651723192, 0.9052418824754537, 0.8846484396889256, 0.865292475366091, 0.8497311645646503, 0.8388691935254817, 0.8256303537004397, 0.8176325372729673, 0.8076852907347736, 0.8000670639426607, 0.7892588692968658, 0.7814415503461742, 0.7731202151361662, 0.7664994567889658, 0.758490072461763, 0.7529625284402992, 0.7485174912671357, 0.7441725498875049 ], "peak_layer": 1, "peak_normalized": 1.7741446897870732, "population_cv": 0.27364872432853743, "rest_mean": 0.00015762320577373935, "spike_contrast": 0.7803181014868814, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012299624067964033, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00027004742878489196, 0.0002509168116375804, 0.0002329077833564952, 0.00021640271006617695, 0.00020191119983792305, 0.00018957215070258826, 0.0001811027032090351, 0.00017255214333999902, 0.00016544139361940324, 0.00016200006939470768, 0.00015566842921543866, 0.0001513263414381072, 0.00014900042151566595, 0.0001447710965294391, 0.00014046284195501357, 0.0001377893495373428, 0.00013465476513374597, 0.00013170854072086513, 0.00012933991092722863, 0.0001276865805266425, 0.00012567146040964872, 0.00012445409083738923, 0.00012293999316170812, 0.00012178040196886286, 0.00012013525702059269, 0.00011894536146428436, 0.00011767874821089208, 0.00011667098442558199, 0.00011545185407157987, 0.00011461049143690616, 0.00011393390013836324, 0.00011327254469506443 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.19015566632151604, "first_to_last_ratio": 1.8431444167877922, "imbalance_abs_log_ratio": 0.6114730352421437, "last_quartile_mean": 0.10316916275769472, "mean": 0.13498378219082952, "normalized": [ 1.7741447141792424, 1.6484611693507762, 1.5301464164964773, 1.421712099711509, 1.3265065258282553, 1.245442086656075, 1.1897999853143832, 1.1336247998774232, 1.0869090145314644, 1.0643003539468234, 1.0227030148964142, 0.9941765578539046, 0.978895844117059, 0.9511102293531465, 0.9228060470726628, 0.9052419251832852, 0.8846484748200404, 0.8652925213936268, 0.8497311905993837, 0.8388692019864461, 0.8256303029992552, 0.8176324960789627, 0.8076852727084025, 0.800067049482784, 0.789258877083647, 0.7814415614650243, 0.7731201949085273, 0.7664994759042474, 0.758490077799444, 0.7529625178867759, 0.7485174642151933, 0.7441725362993381 ], "peak_layer": 1, "peak_normalized": 1.7741447141792424, "population_cv": 0.2736487184871982, "rest_mean": 0.13978183573042904, "spike_contrast": 0.7803180757143806, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.10907429307699204, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.23948076367378235, 0.2225155234336853, 0.20654495060443878, 0.1919080764055252, 0.17905686795711517, 0.16811448335647583, 0.16060370206832886, 0.1530209630727768, 0.14671508967876434, 0.14366328716278076, 0.13804832100868225, 0.13419771194458008, 0.1321350634098053, 0.12838445603847504, 0.12456385046243668, 0.12219297885894775, 0.11941319704055786, 0.116800457239151, 0.11469992995262146, 0.11323373764753342, 0.11144670099020004, 0.11036712676286697, 0.10902441293001175, 0.10799607634544373, 0.10653714835643768, 0.10548193752765656, 0.10435868799686432, 0.1034649983048439, 0.10238385945558548, 0.10163772851228714, 0.10103771835565567, 0.10045122355222702 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 6.883365585963475e-05, "first_to_last_ratio": 2.2497546817252747, "imbalance_abs_log_ratio": 0.8108211799277664, "last_quartile_mean": 3.059607183786284e-05, "mean": 4.691841348858361e-05, "normalized": [ 1.7359303646705315, 1.6475859007944922, 1.5536416135247637, 1.4921074478374423, 1.436565604934822, 1.343834173349448, 1.2787439414688093, 1.2483315192759759, 1.191227587726315, 1.1603086071531024, 1.0995306735006558, 1.059977946939621, 1.0583984121521266, 1.0176934649371145, 0.9754909410255406, 0.9631621800279858, 0.9242301483750994, 0.8752327792179452, 0.8454160845678106, 0.8232909667825862, 0.7863453139125789, 0.77746685672799, 0.7497386589361914, 0.7388508716017261, 0.7093180450413769, 0.6908354487677191, 0.6685471901846501, 0.6592354484095886, 0.631440334976849, 0.6235901145414114, 0.6199506557967245, 0.6139807028410055 ], "peak_layer": 1, "peak_normalized": 1.7359303646705315, "population_cv": 0.3220412433027962, "rest_mean": 4.907019663530971e-05, "spike_contrast": 0.7193528234378936, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.529878449626267e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 8.144709863699973e-05, 7.730211655143648e-05, 7.289439963642508e-05, 7.000731420703232e-05, 6.740137905580923e-05, 6.305056740529835e-05, 5.999663699185476e-05, 5.856973439222202e-05, 5.5890508519951254e-05, 5.4439839004771784e-05, 5.1588234782684594e-05, 4.9732483603293076e-05, 4.965837433701381e-05, 4.774856279254891e-05, 4.576848732540384e-05, 4.519004141911864e-05, 4.3363412260077894e-05, 4.106453343410976e-05, 3.966558142565191e-05, 3.862750600092113e-05, 3.6894074582960457e-05, 3.647751145763323e-05, 3.517654840834439e-05, 3.4665710700210184e-05, 3.328007733216509e-05, 3.241290323785506e-05, 3.136717350571416e-05, 3.0930281354812905e-05, 2.9626178729813546e-05, 2.925785884144716e-05, 2.9087101211189292e-05, 2.8807000489905477e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 0.0001734125216898974, "first_to_last_ratio": 1.495517223041696, "imbalance_abs_log_ratio": 0.4024721155990803, "last_quartile_mean": 0.00011595488103921525, "mean": 0.00013990532806928968, "normalized": [ 1.3768889945483427, 1.3340940553978193, 1.2910049686488183, 1.2527203273458163, 1.2147536525239127, 1.1769854337196524, 1.14744939519443, 1.1220955988146075, 1.0944022472959734, 1.0773022658134643, 1.0508559212107247, 1.03206209473486, 1.0208171898483585, 1.000561882500425, 0.9805161605170362, 0.9663129296065233, 0.9487424991891081, 0.931931984738623, 0.9181974345490227, 0.9075104531838204, 0.8943640941785878, 0.8862664021756573, 0.8757830774574916, 0.86790401991425, 0.8562894022841879, 0.8478155487572495, 0.8387227683262297, 0.8314011142413483, 0.8224071457687988, 0.8161875050523302, 0.8112199678685613, 0.8064334645939688 ], "peak_layer": 1, "peak_normalized": 1.3768889945483427, "population_cv": 0.16325957486062354, "rest_mean": 0.0001431148236776978, "spike_contrast": 0.8564734849544942, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001225740517838858, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0001926341064972803, 0.00018664686649572104, 0.00018061847367789596, 0.00017526224837638438, 0.0001699505082797259, 0.00016466653323732316, 0.00016053428407758474, 0.00015698715287726372, 0.00015311270544771105, 0.00015072032692842185, 0.00014702034241054207, 0.00014439098595175892, 0.00014281776384450495, 0.00013998393842484802, 0.00013717943511437625, 0.00013519232743419707, 0.00013273413060232997, 0.00013038225006312132, 0.00012846071331296116, 0.00012696554767899215, 0.0001251263020094484, 0.00012399339175317436, 0.0001225267187692225, 0.00012142439663875848, 0.00011979944974882528, 0.00011861391249112785, 0.00011734178406186402, 0.00011631744564510882, 0.00011505914153531194, 0.00011418898066040128, 0.00011349399574100971, 0.00011282433843007311 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 0.00014675117381557357, "first_to_last_ratio": 1.2676837465168667, "imbalance_abs_log_ratio": 0.23719141364257146, "last_quartile_mean": 0.00011576323686313117, "mean": 0.00013042020850662084, "normalized": [ 1.169988069421413, 1.156175152925158, 1.1417665260225645, 1.1306726331060184, 1.1183621029407895, 1.1062694829504214, 1.0940115056230622, 1.0844989956600726, 1.0711591662817939, 1.0579765491085218, 1.0443900226505625, 1.035471661013805, 1.0260498632741444, 1.013992390086173, 1.0062479316494695, 0.9977001177294099, 0.9863163473759529, 0.9769959732660659, 0.9687585669786032, 0.9602628595878113, 0.9503845997255937, 0.9426104617668664, 0.9330243108845692, 0.9259760938667744, 0.914797625470175, 0.906289421438531, 0.8956357003805738, 0.888470885240542, 0.8806216558595509, 0.8756991509238615, 0.8712858841028995, 0.8681382926882502 ], "peak_layer": 1, "peak_normalized": 1.169988069421413, "population_cv": 0.09123006364857297, "rest_mean": 0.00013202972385678786, "spike_contrast": 0.9219804606102014, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012172882561571896, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00015259008796419948, 0.00015078860451467335, 0.00014890942838974297, 0.0001474625605624169, 0.00014585701865144074, 0.00014427989663090557, 0.00014268120867200196, 0.00014144058513920754, 0.00013970080181024969, 0.0001379815221298486, 0.00013620956451632082, 0.00013504642993211746, 0.0001338176371064037, 0.00013224509893916547, 0.00013123506505507976, 0.0001301202573813498, 0.00012863558367826045, 0.00012742001854348928, 0.00012634569429792464, 0.00012523768236860633, 0.00012394935765769333, 0.00012293545296415687, 0.00012168522516731173, 0.00012076599523425102, 0.00011930809705518186, 0.00011819845531135798, 0.00011680899478960782, 0.00011587455810513347, 0.00011485085997264832, 0.00011420886585256085, 0.00011363328667357564, 0.00011322277714498341 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00030002388848515693, "first_to_last_ratio": 2.2836575149168885, "imbalance_abs_log_ratio": 0.8257783308640597, "last_quartile_mean": 0.0001313786706305109, "mean": 0.0001947244400071213, "normalized": [ 1.8998833386658438, 1.7835645331795598, 1.6713073472446418, 1.5703972875779748, 1.4780250735507643, 1.3761669788219328, 1.3066447563403325, 1.2401013786965793, 1.1762106459027857, 1.150547637187856, 1.0768735950666006, 1.0264900078881174, 1.0140784159549234, 0.9666871251601796, 0.9096841069789114, 0.8754807117769676, 0.8421876125778792, 0.8073809902734768, 0.77911667677541, 0.7607651063686129, 0.7403822030704592, 0.7308855611321243, 0.7152791523187013, 0.7043381880055872, 0.6944157299089508, 0.686864629891504, 0.680599049276497, 0.6764772714226681, 0.6702412094778135, 0.6672258960044359, 0.6635863560708457, 0.6581114274310631 ], "peak_layer": 1, "peak_normalized": 1.8998833386658438, "population_cv": 0.3592853832280775, "rest_mean": 0.00020492727344390005, "spike_contrast": 0.6813594750078023, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013962913944851608, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0003699537192005664, 0.0003473036049399525, 0.0003254443872720003, 0.00030579473241232336, 0.00028780760476365685, 0.00026797334430739284, 0.0002544356684666127, 0.0002414780465187505, 0.0002290369593538344, 0.00022403974435292184, 0.00020969360775779933, 0.00019988269195891917, 0.0001974658516701311, 0.00018823760910890996, 0.00017713772831484675, 0.00017047749133780599, 0.00016399451124016196, 0.0001572168112033978, 0.00015171305858530104, 0.00014813955931458622, 0.00014417050988413393, 0.00014232128160074353, 0.00013928233238402754, 0.0001371518592350185, 0.00013521971413865685, 0.00013374933041632175, 0.00013252926873974502, 0.00013172665785532445, 0.00013051234418526292, 0.00012992518895771354, 0.00012921648158226162, 0.000128150379168801 ] } } }, "mlp_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "afe29f3ef4d2992353f0bb59fe7c5c2c5050fca671a53184825a8d63cd83a2bc", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.0027073738165199757, 0.002722575329244137, 0.0026850115973502398, 0.0026861224323511124, 0.00271838391199708, 0.0026645169127732515, 0.0026859452482312918, 0.0027359756641089916, 0.0026767742820084095, 0.0027451682835817337, 0.0026999011170119047, 0.002663193503394723, 0.002626648638397455, 0.0026531771291047335, 0.0026517712976783514, 0.002676903735846281, 0.002670086920261383, 0.0026355870068073273, 0.0026610493659973145, 0.0026388936676084995, 0.0026469926815479994, 0.00264862016774714, 0.002641494618728757, 0.002674198243767023, 0.002557571977376938, 0.00252733426168561, 0.0025468147359788418, 0.0026856842450797558, 0.002551089273765683, 0.0025778315030038357, 0.0026001501828432083, 0.0025662274565547705 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.00015858857113926206, "first_to_last_ratio": 3.757867690304466, "imbalance_abs_log_ratio": 1.3238516928952826, "last_quartile_mean": 4.220174423608114e-05, "mean": 8.573221236929385e-05, "normalized": [ 2.5461641896458365, 2.3032502265274117, 2.0924715218207024, 1.8886722264368014, 1.7031663195050915, 1.537434754996766, 1.4223955235407997, 1.3049501874408618, 1.2055297161045848, 1.1692542365736316, 1.0769571962730058, 1.0144505847411098, 0.9913601784485884, 0.9317001386730132, 0.857757082878185, 0.8155634736467603, 0.7682612945963089, 0.7217806418748633, 0.678338533281248, 0.6511820384364709, 0.6192882306865346, 0.6053793146733001, 0.5848721334223365, 0.5718144052517429, 0.5462281883927541, 0.527515083157495, 0.5112993585545178, 0.496920907994256, 0.47722358366421574, 0.46552107689700334, 0.4601234027970099, 0.45317424906679377 ], "peak_layer": 1, "peak_normalized": 2.5461641896458365, "population_cv": 0.5686091737198813, "rest_mean": 9.231269224848668e-05, "spike_contrast": 0.5437781067692293, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.0197621021652594e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00021828828903380781, 0.00019746273756027222, 0.00017939221288543195, 0.0001619200484128669, 0.00014601621660403907, 0.000131807682919316, 0.00012194511509733275, 0.00011187626660102978, 0.00010335272963857278, 0.00010024275252362713, 9.232992306351662e-05, 8.697109296917915e-05, 8.499150135321543e-05, 7.987671415321529e-05, 7.353741239057854e-05, 6.992006092332304e-05, 6.586474046343938e-05, 6.187985127326101e-05, 5.8155463193543255e-05, 5.582727681030519e-05, 5.309295011102222e-05, 5.1900507969548926e-05, 5.0142381951445714e-05, 4.902291402686387e-05, 4.682935104938224e-05, 4.522503513726406e-05, 4.3834825191879645e-05, 4.260212881490588e-05, 4.0913433622336015e-05, 3.991015182691626e-05, 3.944739728467539e-05, 3.885163096128963e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.00021242323236947414, "first_to_last_ratio": 1.8345651540958352, "imbalance_abs_log_ratio": 0.6068074801791611, "last_quartile_mean": 0.0001157894184871111, "mean": 0.00015111673860701558, "normalized": [ 1.771707915810251, 1.6366668487510905, 1.5242909365839221, 1.4186705636795862, 1.3264449439106374, 1.2453857912414694, 1.1893415085703798, 1.1330085309323608, 1.0858463851390703, 1.062861239127963, 1.023237806768965, 0.9942126959777651, 0.978436257513136, 0.9531027445378829, 0.9232414968561398, 0.904013237249231, 0.8853704564278231, 0.8680164030054869, 0.8515402788738432, 0.8395817783841374, 0.8267435189825344, 0.8185143642831039, 0.809172703595369, 0.8007918347797284, 0.7909896391756025, 0.7821052393452608, 0.7753388667989797, 0.7682453290925442, 0.7604995795745633, 0.7549901567959042, 0.7508572350862155, 0.7467737131490535 ], "peak_layer": 1, "peak_normalized": 1.771707915810251, "population_cv": 0.2714170004285523, "rest_mean": 0.00015645500981783562, "spike_contrast": 0.7816309251520459, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012229007406858728, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0002677347220014781, 0.00024732775636948645, 0.00023034587502479553, 0.00021438486874103546, 0.00020044803386554122, 0.0001881986390799284, 0.00017972940986510366, 0.00017121655400842428, 0.00016408956435043365, 0.00016061612404882908, 0.0001546283601783216, 0.00015024218009784818, 0.00014785809617023915, 0.0001440297783119604, 0.00013951724395155907, 0.000136611532070674, 0.0001337942958343774, 0.00013117180787958205, 0.0001286819897359237, 0.00012687486014328897, 0.00012493478425312787, 0.00012369122123345733, 0.00012227953993715346, 0.00012101305037504062, 0.00011953177454415709, 0.00011818919301731512, 0.00011716668086592108, 0.00011609472858253866, 0.00011492421617731452, 0.00011409165017539635, 0.00011346709652571008, 0.00011285000800853595 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.18837904930114746, "first_to_last_ratio": 1.8345651581276443, "imbalance_abs_log_ratio": 0.6068074823768531, "last_quartile_mean": 0.10268321540206671, "mean": 0.13401183625683188, "normalized": [ 1.7717078885231814, 1.6366668119471057, 1.524290957183596, 1.418670514273678, 1.326444920903592, 1.2453858718827995, 1.1893415500975306, 1.1330085716119305, 1.0858463365841577, 1.062861212873729, 1.0232377434953237, 0.9942127340740732, 0.9784362443707608, 0.9531027271661117, 0.9232414917376033, 0.904013241653558, 0.885370477646398, 0.8680164390463408, 0.8515402681847718, 0.8395816969955481, 0.8267435345782925, 0.8185144279720658, 0.8091727254708025, 0.8007918405918669, 0.7909896328038464, 0.7821052665924377, 0.7753388466443417, 0.7682453528580236, 0.7604996016741158, 0.7549901615676585, 0.7508572336421532, 0.7467736753526057 ], "peak_layer": 1, "peak_normalized": 1.7717078885231814, "population_cv": 0.27141699472708714, "rest_mean": 0.1387458680956452, "spike_contrast": 0.7816309473985974, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.10844806432724, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.23742982745170593, 0.2193327248096466, 0.2042730301618576, 0.19011864066123962, 0.1777593195438385, 0.16689644753932953, 0.15938584506511688, 0.151836559176445, 0.1455162614583969, 0.1424359828233719, 0.13712596893310547, 0.13323627412319183, 0.13112203776836395, 0.12772704660892487, 0.1237252876162529, 0.1211484745144844, 0.11865012347698212, 0.11632447689771652, 0.11411647498607635, 0.11251388490200043, 0.11079341918230057, 0.10969062149524689, 0.10843872278928757, 0.10731558501720428, 0.10600197315216064, 0.10481136292219162, 0.10390458256006241, 0.1029539704322815, 0.10191594809293747, 0.10117761790752411, 0.10062375664710999, 0.10007651150226593 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 6.873187476230669e-05, "first_to_last_ratio": 2.248757500145603, "imbalance_abs_log_ratio": 0.8103778415280068, "last_quartile_mean": 3.0564378221242805e-05, "mean": 4.685761763312257e-05, "normalized": [ 1.7400006587426948, 1.6405194725375354, 1.5551306973978087, 1.4944078341288562, 1.4327724019957897, 1.3414187773842097, 1.2804088050749736, 1.2499327782691996, 1.1927955171655855, 1.1608931065609767, 1.0981756972550076, 1.0577423904261343, 1.0542786038595535, 1.0218640087393278, 0.9775807389332813, 0.9570972402588515, 0.9209982861589985, 0.8815092249082567, 0.8478232134430647, 0.8204952153707635, 0.7871780668180614, 0.7759097745574741, 0.7506963524238334, 0.7421155343193898, 0.7072927325438048, 0.6891814936654451, 0.6695399899996352, 0.6616343983211914, 0.6345111343049281, 0.6212687927046295, 0.6200231913288072, 0.6148038704019305 ], "peak_layer": 1, "peak_normalized": 1.7400006587426948, "population_cv": 0.3217536161653951, "rest_mean": 4.900405669845727e-05, "spike_contrast": 0.719671983430414, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.526684668031521e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 8.153228554874659e-05, 7.687083416385576e-05, 7.286971958819777e-05, 7.002439087955281e-05, 6.71363013680093e-05, 6.285568815656006e-05, 5.999690620228648e-05, 5.8568872191244736e-05, 5.5891556257847697e-05, 5.439668530016206e-05, 5.145789691596292e-05, 4.9563288484932855e-05, 4.9400983698433265e-05, 4.7882112994557247e-05, 4.580710447044112e-05, 4.484729652176611e-05, 4.3155785533599555e-05, 4.130542220082134e-05, 3.972697595600039e-05, 3.844645107164979e-05, 3.688528886414133e-05, 3.635728353401646e-05, 3.517584264045581e-05, 3.477376594673842e-05, 3.314205241622403e-05, 3.229340290999971e-05, 3.1373048841487616e-05, 3.10026116494555e-05, 2.9731680115219206e-05, 2.9111175535945222e-05, 2.9052809622953646e-05, 2.8808244678657502e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 0.0001719075116852764, "first_to_last_ratio": 1.4894635014227988, "imbalance_abs_log_ratio": 0.39841598895878805, "last_quartile_mean": 0.00011541572621354135, "mean": 0.00013894635094402474, "normalized": [ 1.3738985779819257, 1.3296082759737424, 1.2883384004275333, 1.2499940655466777, 1.2130730176151365, 1.1755503979222366, 1.147409009170091, 1.11990595257071, 1.0929037170303153, 1.075598370036652, 1.0512762916941902, 1.0312594730951, 1.0194418974669357, 1.0018996501730997, 0.9803460168062212, 0.9648004557293595, 0.9493010807852901, 0.9344788929945884, 0.9198953859218769, 0.908001147597105, 0.8952913731815082, 0.886953676542665, 0.8771894463102521, 0.8683888934041172, 0.8579399877904575, 0.8482799643655919, 0.8408636339472488, 0.8330870307470335, 0.82434015220157, 0.8181187963300758, 0.8135579418994486, 0.809009026741244 ], "peak_layer": 1, "peak_normalized": 1.3738985779819257, "population_cv": 0.16161423472023242, "rest_mean": 0.0001421073115866252, "spike_contrast": 0.8576417505420824, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012187716347398237, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0001908981939777732, 0.0001847442181315273, 0.00017900991952046752, 0.00017368211410939693, 0.00016855206922627985, 0.00016333843814209104, 0.00015942829486448318, 0.00015560684551019222, 0.0001518549834145233, 0.00014945046859793365, 0.00014607100456487387, 0.0001432897406630218, 0.00014164773165248334, 0.00013921030040364712, 0.000136215501697734, 0.0001340555027127266, 0.0001319019211223349, 0.00012984243221580982, 0.0001278161071240902, 0.00012616344611160457, 0.00012439746933523566, 0.00012323897681199014, 0.00012188227265141904, 0.00012065946793882176, 0.00011920763063244522, 0.00011786540562752634, 0.00011683493357850239, 0.00011575440294109285, 0.0001145390560850501, 0.00011367462138878182, 0.00011304090730845928, 0.00011240885214647278 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 0.00014552221000485588, "first_to_last_ratio": 1.262697055715921, "imbalance_abs_log_ratio": 0.2332499537229619, "last_quartile_mean": 0.00011524712863320019, "mean": 0.00012953460168318998, "normalized": [ 1.1676213631634658, 1.1525055688027706, 1.1391580058081985, 1.1289307982409096, 1.1166731577821778, 1.1054001772932092, 1.094813144753513, 1.0822854389705017, 1.0679308603491235, 1.0564894822224282, 1.0456838360716472, 1.0337072702557388, 1.0234945545465333, 1.0158819477934218, 1.0051772952892828, 0.9951467950359763, 0.9866643397779161, 0.9787559418439278, 0.970448736962688, 0.9607908684400951, 0.9509939230158972, 0.9428061355388626, 0.934924980051271, 0.9261034265480621, 0.9170582603044295, 0.9058562786860701, 0.8981778407016658, 0.8907564927732574, 0.8831168120837269, 0.8778216107141017, 0.8745192645108896, 0.8703053916682415 ], "peak_layer": 1, "peak_normalized": 1.1676213631634658, "population_cv": 0.08967236486753827, "rest_mean": 0.00013110875065817877, "spike_contrast": 0.9231588022206524, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012103419721825049, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00015124736819416285, 0.00014928934979252517, 0.000147560378536582, 0.00014623560127802193, 0.00014464781270362437, 0.00014318757166620344, 0.00014181618462316692, 0.00014019341324456036, 0.0001383339986205101, 0.00013685194426216185, 0.00013545223919209093, 0.00013390085950959474, 0.00013257795944809914, 0.00013159186346456409, 0.00013020524056628346, 0.00012890594371128827, 0.00012780717224813998, 0.00012678276107180864, 0.0001257066905964166, 0.0001244556624442339, 0.00012318661902099848, 0.0001221260172314942, 0.00012110513489460573, 0.0001199624384753406, 0.00011879077646881342, 0.00011733973224181682, 0.00011634510883595794, 0.00011538378748809919, 0.0001143941844929941, 0.00011370827269274741, 0.00011328050459269434, 0.0001127346622524783 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00029741139769612346, "first_to_last_ratio": 2.278072415841521, "imbalance_abs_log_ratio": 0.8233296537390045, "last_quartile_mean": 0.00013055396993877366, "mean": 0.00019328556300024502, "normalized": [ 1.889398576482247, 1.780233894901915, 1.6772594716902607, 1.566224958959574, 1.4762232610465842, 1.375571144634086, 1.3083686481730985, 1.2364401527267181, 1.173148897934763, 1.1508864184295666, 1.080457040307468, 1.0318582947152708, 1.0081957004611959, 0.9698994715426978, 0.9066127339785177, 0.8754550048258108, 0.8446146862114313, 0.8115724449181009, 0.7806677558079268, 0.7613480996239201, 0.7403134026279838, 0.7294052762839086, 0.7160186976339193, 0.7062575707090604, 0.6942777320753531, 0.6866005528973866, 0.6829938476675228, 0.6753403086482458, 0.6715967314167978, 0.6686860556999406, 0.6640286432895989, 0.6600445236791291 ], "peak_layer": 1, "peak_normalized": 1.889398576482247, "population_cv": 0.3579154260008552, "rest_mean": 0.00020340604748559632, "spike_contrast": 0.681567477924495, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013863494677934796, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0003651934675872326, 0.0003440935106482357, 0.00032419004128314555, 0.0003027286729775369, 0.0002853326441254467, 0.00026587804313749075, 0.00025288877077400684, 0.00023898603103589267, 0.00022675274522043765, 0.00022244972933549434, 0.0002088367473334074, 0.00019944331143051386, 0.00019486967357806861, 0.00018746756541077048, 0.00017523515271022916, 0.00016921281348913908, 0.0001632518251426518, 0.00015686523693148047, 0.00015089180669747293, 0.00014715759607497603, 0.00014309189282357693, 0.00014098350948188454, 0.00013839607709087431, 0.0001365093921776861, 0.0001341938623227179, 0.00013270997442305088, 0.00013201285037212074, 0.00013053353177383542, 0.0001298099523410201, 0.00012924736074637622, 0.00012834715016651899, 0.00012757707736454904 ] } } }, "post_attention_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "31853ffb07b24cc59e774bb20656840b4358b3080b42516d380f2dc0f46404c8", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.0022265110164880753, 0.0040087271481752396, 0.005379301495850086, 0.006410556845366955, 0.0027253651060163975, 0.004838948138058186, 0.006175771821290255, 0.00742313964292407, 0.0030937574338167906, 0.005150026176124811, 0.007019025273621082, 0.008143599145114422, 0.004255065228790045, 0.0066224560141563416, 0.008129785768687725, 0.010099578648805618, 0.0041999719105660915, 0.00719039561226964, 0.009030314162373543, 0.010736005380749702, 0.005131533369421959, 0.00787449348717928, 0.009912566281855106, 0.012113703414797783, 0.0059690922498703, 0.00876442901790142, 0.010806876234710217, 0.012353328987956047, 0.006502739619463682, 0.009470963850617409, 0.012123330496251583, 0.01384514570236206 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.00015962236011546338, "first_to_last_ratio": 3.7698422272252903, "imbalance_abs_log_ratio": 1.3270331510418396, "last_quartile_mean": 4.2341920561739244e-05, "mean": 8.61727822893954e-05, "normalized": [ 2.542734798705971, 2.3221448837042376, 2.101145461274622, 1.8911858524935854, 1.7004204056817789, 1.5354142426015858, 1.4229001770142031, 1.3028733810328659, 1.2049412416934622, 1.1705258152816296, 1.0773908232649334, 1.0133220774376726, 0.990609360100461, 0.9272534949358001, 0.8550385909364457, 0.8185651575045715, 0.7703972051466339, 0.7173947009730917, 0.6760256695667757, 0.6514523109028573, 0.618419077603279, 0.6053620388084041, 0.5838930841859182, 0.5697043695032278, 0.5457802700901755, 0.5276149005446632, 0.5104701332743111, 0.49517988582787786, 0.47527562953524777, 0.46579059316028215, 0.45896435764104826, 0.45181000957238127 ], "peak_layer": 1, "peak_normalized": 2.542734798705971, "population_cv": 0.5705202801130907, "rest_mean": 9.280119640472414e-05, "spike_contrast": 0.542873885449774, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.03793460666202e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00021911453222855926, 0.00020010568550787866, 0.00018106155039276928, 0.00016296874673571438, 0.00014652995741926134, 0.00013231091725174338, 0.0001226152671733871, 0.00011227222421439365, 0.00010383313929196447, 0.00010086746624438092, 9.284176485380158e-05, 8.732078276807442e-05, 8.536356472177431e-05, 7.99040135461837e-05, 7.368105434579775e-05, 7.053803710732609e-05, 6.638727063545957e-05, 6.181989738252014e-05, 5.825501284562051e-05, 5.613745815935545e-05, 5.329089253791608e-05, 5.216573117650114e-05, 5.0315691623836756e-05, 4.9093010602518916e-05, 4.703140439232811e-05, 4.546604395727627e-05, 4.398863165988587e-05, 4.267102849553339e-05, 4.095582335139625e-05, 4.013847137684934e-05, 3.955023566959426e-05, 3.893372559105046e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.00021442477373057045, "first_to_last_ratio": 1.8431254562763404, "imbalance_abs_log_ratio": 0.6114627481432037, "last_quartile_mean": 0.00011633759004325839, "mean": 0.00015221248349917005, "normalized": [ 1.7740805953138137, 1.6484457246979003, 1.5301434809496548, 1.4217241600875643, 1.3264958046529274, 1.2454577490392524, 1.1897861330408546, 1.1336265611690741, 1.086914920172498, 1.064301358569379, 1.0227209980571423, 0.9941753882589759, 0.9788979908211966, 0.9511107967643203, 0.9228143274453928, 0.9052447611165039, 0.8846530997418666, 0.8652930871753776, 0.8497381552743479, 0.8388656494639406, 0.8256334792987466, 0.8176309647342147, 0.8076951735882674, 0.8000661307359944, 0.7892605467734257, 0.78144613040264, 0.7731219607566042, 0.7664988488799842, 0.7584935618388989, 0.7529639051546407, 0.7485195296370439, 0.7441790263875565 ], "peak_layer": 1, "peak_normalized": 1.7740805953138137, "population_cv": 0.2736411578809574, "rest_mean": 0.00015762286892384981, "spike_contrast": 0.7803207938393821, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012299640220589936, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00027003721334040165, 0.00025091401766985655, 0.00023290693934541196, 0.0002164041652576998, 0.000201909220777452, 0.0001895742170745507, 0.00018110030214302242, 0.00017255211423616856, 0.00016544201935175806, 0.00016199995297938585, 0.00015567090304102749, 0.00015132590488065034, 0.0001490004942752421, 0.00014477093645837158, 0.00014046386058907956, 0.00013778955326415598, 0.0001346552453469485, 0.00013170840975362808, 0.00012934075493831187, 0.00012768582382705063, 0.00012567172234412283, 0.00012445363972801715, 0.00012294128828216344, 0.00012178005272289738, 0.00012013530795229599, 0.00011894585622940212, 0.00011767881369451061, 0.00011667069338727742, 0.00011545218876563013, 0.00011461050598882139, 0.00011393401655368507, 0.00011327333777444437 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.190154030919075, "first_to_last_ratio": 1.84312545379361, "imbalance_abs_log_ratio": 0.6114627467961817, "last_quartile_mean": 0.10316933691501617, "mean": 0.13498355145566165, "normalized": [ 1.774080628234443, 1.6484456620184311, 1.5301434020504485, 1.4217242444591272, 1.326495766996536, 1.2454576834375484, 1.189786122599316, 1.133626627255238, 1.0869149569673349, 1.0643014004698326, 1.0227209907477024, 0.9941753870542451, 0.9788979589698585, 0.9511107512164827, 0.9228143032197887, 0.9052447972745712, 0.8846531331847195, 0.8652931173486506, 0.8497382179112226, 0.8388656682549475, 0.8256334805754935, 0.8176309131099365, 0.8076951535479161, 0.8000661540388709, 0.7892605573866218, 0.7814461538045975, 0.7731219580171443, 0.7664988542586116, 0.7584935821787124, 0.7529638601652359, 0.7485195164462104, 0.7441789968002049 ], "peak_layer": 1, "peak_normalized": 1.774080628234443, "population_cv": 0.27364115345546325, "rest_mean": 0.13978153549962574, "spike_contrast": 0.7803207857775153, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.10907443761825561, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.2394717037677765, 0.22251304984092712, 0.2065441906452179, 0.19190938770771027, 0.1790551096200943, 0.16811630129814148, 0.16060155630111694, 0.1530209481716156, 0.14671564102172852, 0.1436631828546524, 0.13805051147937775, 0.13419732451438904, 0.13213512301445007, 0.1283843070268631, 0.1245647519826889, 0.12219315767288208, 0.11941362172365189, 0.11680033802986145, 0.11470068246126175, 0.1132330670952797, 0.11144693940877914, 0.11036672443151474, 0.10902556031942368, 0.10799577087163925, 0.10653719305992126, 0.10548237711191177, 0.1043587476015091, 0.10346473753452301, 0.10238415747880936, 0.10163773596286774, 0.10103782266378403, 0.10045192390680313 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 6.883367768750759e-05, "first_to_last_ratio": 2.249759357551583, "imbalance_abs_log_ratio": 0.810823258297238, "last_quartile_mean": 3.059601795030176e-05, "mean": 4.691846504556452e-05, "normalized": [ 1.7359059710084281, 1.6475789727938472, 1.5536456441204052, 1.4921194549590373, 1.4365657321885223, 1.3438513058553838, 1.2787343947807137, 1.2483299149152907, 1.1912354282513808, 1.1603083401306369, 1.099547144002948, 1.059972440022979, 1.0583955432739494, 1.0176800955770247, 0.9754963047746087, 0.9631595708780456, 0.9242400656751482, 0.8752301116137658, 0.8454245377074081, 0.8232853322610043, 0.7863494898171388, 0.7774642190168813, 0.7497486129019063, 0.7388483538626245, 0.709315327139865, 0.6908343794807076, 0.6685469983108738, 0.6592365073812129, 0.6314341746588822, 0.6235898169926835, 0.6199474545603838, 0.6139783610863121 ], "peak_layer": 1, "peak_normalized": 1.7359059710084281, "population_cv": 0.3220406003070692, "rest_mean": 4.9070239684725595e-05, "spike_contrast": 0.7193541792517962, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.52988819940947e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 8.144604362314567e-05, 7.730187644483522e-05, 7.28946688468568e-05, 7.000795449130237e-05, 6.740145909134299e-05, 6.305144052021205e-05, 5.999625500408001e-05, 5.8569723478285596e-05, 5.589093780145049e-05, 5.4439886298496276e-05, 5.158906424185261e-05, 4.973227987647988e-05, 4.9658294301480055e-05, 4.774798799189739e-05, 4.576878927764483e-05, 4.51899686595425e-05, 4.3363925215089694e-05, 4.1064453398576006e-05, 3.9666021621087566e-05, 3.86272840842139e-05, 3.689431105158292e-05, 3.6477427784120664e-05, 3.5177054087398574e-05, 3.466563066467643e-05, 3.3279986382694915e-05, 3.2412888685939834e-05, 3.136719897156581e-05, 3.093036502832547e-05, 2.9625922252307646e-05, 2.9257877031341195e-05, 2.9086982976878062e-05, 2.8806922273361124e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 0.00017341253987979144, "first_to_last_ratio": 1.4955148931299893, "imbalance_abs_log_ratio": 0.4024705576674927, "last_quartile_mean": 0.00011595507385209203, "mean": 0.00013990553861731314, "normalized": [ 1.3768818258178377, 1.3340879911950096, 1.2910054180628048, 1.2527236427111486, 1.2147542166907819, 1.1769889670731195, 1.1474411155803916, 1.1220953663134672, 1.0944075691302455, 1.0773007485615413, 1.0508678613623372, 1.0320596054419082, 1.0208161736522385, 1.000559648637946, 0.9805197815181176, 0.9663128275349431, 0.948744191771126, 0.9319305822463463, 0.9182019814348666, 0.9075038868226692, 0.8943650364968407, 0.886262676120959, 0.8757907045344768, 0.8679002174812178, 0.856288373659771, 0.8478178092769747, 0.8387215581135022, 0.8313975747674917, 0.8224084044020752, 0.8161864847722404, 0.8112200992010112, 0.8064376596145924 ], "peak_layer": 1, "peak_normalized": 1.3768818258178377, "population_cv": 0.16325888721442564, "rest_mean": 0.0001431150389921472, "spike_contrast": 0.8564734877369171, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001225742365932092, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0001926333934534341, 0.00018664629897102714, 0.00018061880837194622, 0.0001752629759721458, 0.00016995084297377616, 0.0001646672753849998, 0.00016053336730692536, 0.00015698735660407692, 0.00015311368042603135, 0.00015072034148033708, 0.00014702223415952176, 0.00014439085498452187, 0.0001428178366040811, 0.00013998383656144142, 0.00013718014815822244, 0.00013519251660909504, 0.00013273456715978682, 0.00013038225006312132, 0.00012846154277212918, 0.00012696482008323073, 0.00012512662215158343, 0.0001239930570591241, 0.00012252797023393214, 0.000121424047392793, 0.00011979948612861335, 0.00011861440725624561, 0.00011734179133782163, 0.0001163171255029738, 0.00011505949078127742, 0.00011418900976423174, 0.00011349418491590768, 0.00011282509512966499 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 0.00014675183047074825, "first_to_last_ratio": 1.2676545313577883, "imbalance_abs_log_ratio": 0.23716836728277432, "last_quartile_mean": 0.00011576642282307148, "mean": 0.00013041823922321782, "normalized": [ 1.1700698937843759, 1.1560440994564782, 1.1417398043557874, 1.130463200949706, 1.118403537277814, 1.1065304334184665, 1.0939881912841023, 1.0846815122144253, 1.0709468270677047, 1.0579907390333845, 1.0442838370166418, 1.035550673160169, 1.025959021745254, 1.0140203095307772, 1.0062047700154308, 0.9976631870190885, 0.9861765922450596, 0.9770016877798172, 0.9687702939454803, 0.9602422119882037, 0.9504995944091904, 0.9425883202531964, 0.9330014667281973, 0.9259385264566372, 0.9147443798098084, 0.906202796813466, 0.8958065504175365, 0.8885655303249116, 0.8806194435723184, 0.8757905905296257, 0.8713783728577705, 0.8681336045391749 ], "peak_layer": 1, "peak_normalized": 1.1700698937843759, "population_cv": 0.09121429133712369, "rest_mean": 0.0001320278307604086, "spike_contrast": 0.921975648780332, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001217264449223876, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0001525984553154558, 0.0001507692359155044, 0.000148903694935143, 0.00014743302017450333, 0.00014586022007279098, 0.00014431175077334046, 0.00014267601363826543, 0.00014146225294098258, 0.000139670999487862, 0.00013798128929920495, 0.00013619365927297622, 0.00013505469541996717, 0.00013380376913119107, 0.00013224674330558628, 0.00013122745440341532, 0.00013011347618885338, 0.00012861541472375393, 0.00012741883983835578, 0.0001263453159481287, 0.0001252330985153094, 0.0001239624834852293, 0.00012293070903979242, 0.00012168040848337114, 0.00012075927224941552, 0.00011929935135412961, 0.00011818537313956767, 0.00011682951299007982, 0.00011588515189941972, 0.00011484883725643158, 0.00011421906674513593, 0.00011364363308530301, 0.00011322045611450449 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00030004180916876066, "first_to_last_ratio": 2.2837506327398005, "imbalance_abs_log_ratio": 0.8258191057718037, "last_quartile_mean": 0.00013138116082700435, "mean": 0.00019473525253488333, "normalized": [ 1.9000096511818072, 1.783700741635724, 1.6708066910504589, 1.570366884717585, 1.4783058799917985, 1.376185321596543, 1.3064973297630273, 1.2402700043336092, 1.1767953849206394, 1.1505341943958838, 1.0771542571420398, 1.026098162637436, 1.013898063759585, 0.9669473025647412, 0.9095398902231254, 0.8755205777387188, 0.842301214161826, 0.8073124727024382, 0.7791991818813049, 0.760574607765185, 0.7402055397447304, 0.7308143413325056, 0.7152667122155245, 0.7043714155678553, 0.6942467749765324, 0.6868749151283189, 0.680655564574882, 0.676363713518914, 0.6701061029149729, 0.6672276319278244, 0.6636177363814285, 0.6582317375530352 ], "peak_layer": 1, "peak_normalized": 1.9000096511818072, "population_cv": 0.35930746026536914, "rest_mean": 0.00020494150898971216, "spike_contrast": 0.6812747128050888, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013962146767880766, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0003699988592416048, 0.0003473494143690914, 0.0003253649629186839, 0.0003058057918678969, 0.0002878782688640058, 0.0002679917961359024, 0.0002544210874475539, 0.00024152429250534624, 0.00022916354646440595, 0.000224049566895701, 0.00020975990628357977, 0.00019981748482678086, 0.000197441695490852, 0.0001882987271528691, 0.00017711948021315038, 0.00017049472080543637, 0.00016402573965024203, 0.00015721219824627042, 0.00015173754945863038, 0.00014811068831477314, 0.00014414411270990968, 0.0001423153153154999, 0.0001392876438330859, 0.00013716594548895955, 0.00013519432104658335, 0.00013375876005738974, 0.00013254763325676322, 0.0001317118585575372, 0.00013049328117631376, 0.00012993274140171707, 0.00012922976748086512, 0.00012818092363886535 ] } } }, "post_mlp_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "7fd9c911e93175896ad618842c120e77b3f017c17b7eec9c2bfb01fc6dbc8ac2", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.0034885562490671873, 0.004831995815038681, 0.005952076520770788, 0.006925336085259914, 0.00385080324485898, 0.005542940925806761, 0.006729288958013058, 0.007927131839096546, 0.004081308841705322, 0.005811207927763462, 0.007534808944910765, 0.008581922389566898, 0.00499920267611742, 0.007102519739419222, 0.008542952127754688, 0.010447698645293713, 0.005041283555328846, 0.007636935915797949, 0.009387043304741383, 0.011020516976714134, 0.005807018838822842, 0.008367395959794521, 0.01024632342159748, 0.012378672137856483, 0.006453148555010557, 0.009127611294388771, 0.011037957854568958, 0.012611051090061665, 0.007010402157902718, 0.009794782847166061, 0.012445803731679916, 0.014087481424212456 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.00015858902861509705, "first_to_last_ratio": 3.757869257587208, "imbalance_abs_log_ratio": 1.3238521099622325, "last_quartile_mean": 4.220184837322449e-05, "mean": 8.57325238712292e-05, "normalized": [ 2.5462257183789148, 2.3032637538343064, 2.0924289533379263, 1.8886492391629932, 1.7031256747388024, 1.5374176267930122, 1.4224231993460965, 1.3049597038569594, 1.2055618292025094, 1.169245405308266, 1.0769812048488787, 1.0144334047898542, 0.9913512297333369, 0.9317090592952362, 0.8577499774884012, 0.81555804918997, 0.7682725064221458, 0.7217807351250844, 0.6783324192661732, 0.6511800118936032, 0.6192916242842709, 0.6053728292385717, 0.5848720876072855, 0.5718124973482859, 0.5462302773879167, 0.5275173250132701, 0.5112998770983732, 0.49691833866078555, 0.47721658788927634, 0.46551989467389665, 0.46012635628943105, 0.45317260249616614 ], "peak_layer": 1, "peak_normalized": 2.5462257183789148, "population_cv": 0.5686111305901927, "rest_mean": 9.231303704803361e-05, "spike_contrast": 0.5437775023084314, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.019775271648541e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00021829435718245804, 0.00019746461475733668, 0.0001793892151908949, 0.0001619186659809202, 0.00014601326256524771, 0.00013180669338908046, 0.00012194793089292943, 0.00011187748896190897, 0.00010335585830034688, 0.00010024235962191597, 9.233231685357168e-05, 8.69699360919185e-05, 8.499104296788573e-05, 7.987776916706935e-05, 7.353707042057067e-05, 6.991984992055222e-05, 6.586594099644572e-05, 6.188008410390466e-05, 5.8155150327365845e-05, 5.582730591413565e-05, 5.309343396220356e-05, 5.190014053368941e-05, 5.014256021240726e-05, 4.90229285787791e-05, 4.682970029534772e-05, 4.522539165918715e-05, 4.383502891869284e-05, 4.2602063331287354e-05, 4.0912982512963936e-05, 3.991019548266195e-05, 3.944779382436536e-05, 3.885163096128963e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.00021242368711682502, "first_to_last_ratio": 1.8345658968465297, "imbalance_abs_log_ratio": 0.6068078850437921, "last_quartile_mean": 0.0001157896194854402, "mean": 0.0001511171253696375, "normalized": [ 1.7717451736709768, 1.636674793183293, 1.5242755762025908, 1.4186515254985965, 1.3264266232459345, 1.2453738409548603, 1.1893512719369177, 1.1330135273991528, 1.085860457802012, 1.0628542818802846, 1.023252039669589, 0.9942070699734702, 0.9784308644775869, 0.9531083940389613, 0.9232390376562555, 0.9040072643238835, 0.8853775311228679, 0.8680181295617843, 0.8515367513400655, 0.8395795332993111, 0.8267439067367663, 0.8185133286645813, 0.8091730400239778, 0.800791325997512, 0.7909899739933552, 0.7821072339266707, 0.7753403490721149, 0.7682422073314642, 0.7604971517068638, 0.7549863467424895, 0.7508596466770359, 0.7467718018887739 ], "peak_layer": 1, "peak_normalized": 1.7717451736709768, "population_cv": 0.2714186609524326, "rest_mean": 0.00015645541295978345, "spike_contrast": 0.7816308178118705, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012229037238284946, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0002677410375326872, 0.0002473295899108052, 0.00023034414334688336, 0.00021438254043459892, 0.00020044577831868082, 0.00018819731485564262, 0.00017973134526982903, 0.000171217747265473, 0.0001640921109355986, 0.00016061548376455903, 0.00015463090676348656, 0.00015024171443656087, 0.0001478576596127823, 0.00014403100067283958, 0.00013951722939964384, 0.00013661097909789532, 0.00013379570737015456, 0.0001311724045081064, 0.0001286817860091105, 0.00012687484559137374, 0.0001249351626029238, 0.00012369138130452484, 0.00012227990373503417, 0.00012101328320568427, 0.00011953213106608018, 0.0001181897969217971, 0.0001171672047348693, 0.00011609455395955592, 0.00011492414341773838, 0.0001140913664130494, 0.00011346775136189535, 0.00011285000800853595 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.18837944976985455, "first_to_last_ratio": 1.8345658800697515, "imbalance_abs_log_ratio": 0.6068078758989703, "last_quartile_mean": 0.10268339328467846, "mean": 0.1340121782850474, "normalized": [ 1.7717451751532867, 1.6366747548098712, 1.5242756140248972, 1.4186514377480737, 1.3264266357207033, 1.2453739091726768, 1.1893513017871604, 1.1330134634090196, 1.0858604665419038, 1.0628542749024097, 1.0232520332438115, 0.9942070832344971, 0.9784308561813967, 0.9531084116998004, 0.9232390798282929, 0.9040072094634526, 0.8853775581660035, 0.8680181154203164, 0.8515367049614931, 0.8395794986042229, 0.8267439263823189, 0.8185133396785964, 0.809173050926764, 0.8007913534932437, 0.7909900046670136, 0.782107217826699, 0.7753403703776178, 0.7682422246066585, 0.7604971603471317, 0.7549863443945886, 0.7508596538022533, 0.7467717694238248 ], "peak_layer": 1, "peak_normalized": 1.7717451751532867, "population_cv": 0.27141865758333467, "rest_mean": 0.13874622461972413, "spike_contrast": 0.7816308398662988, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.10844832807779312, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.23743543028831482, 0.21933434903621674, 0.20427149534225464, 0.19011656939983368, 0.17775732278823853, 0.16689527034759521, 0.15938755869865417, 0.15183760225772858, 0.14551852643489838, 0.14243541657924652, 0.13712823390960693, 0.1332358568906784, 0.1311216503381729, 0.12772813439369202, 0.1237252801656723, 0.12114797532558441, 0.1186513751745224, 0.1163249984383583, 0.11411628872156143, 0.11251387745141983, 0.11079375445842743, 0.10969075560569763, 0.10843904316425323, 0.107315793633461, 0.10600229352712631, 0.104811891913414, 0.10390505194664001, 0.10295381397008896, 0.1019158810377121, 0.10117736458778381, 0.10062433779239655, 0.10007651150226593 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 6.873207303215167e-05, "first_to_last_ratio": 2.248757814140914, "imbalance_abs_log_ratio": 0.8103779811585758, "last_quartile_mean": 3.0564462122129044e-05, "mean": 4.6857780716891284e-05, "normalized": [ 1.7400119939228862, 1.6405409364325143, 1.555121247708426, 1.4944054279879166, 1.4327475398601994, 1.3414100714975594, 1.2804101716466747, 1.2499370459040815, 1.192804641985012, 1.1608858830017554, 1.098191983597808, 1.057744066133373, 1.0542717513613309, 1.021869458338179, 0.9775749297615711, 0.9570927445976433, 0.9210091333285576, 0.8815125232807631, 0.847820728512489, 0.8204970180441978, 0.7871838673851995, 0.7759041238100245, 0.7506938949785397, 0.7421070509174572, 0.7072987335050381, 0.6891811912847667, 0.6695407652827761, 0.6616333377893114, 0.6345027924930273, 0.6212629426015038, 0.6200302724102036, 0.614801730639213 ], "peak_layer": 1, "peak_normalized": 1.7400119939228862, "population_cv": 0.32175451551289735, "rest_mean": 4.9004235565748616e-05, "spike_contrast": 0.7196709452950106, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.5266924533061685e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 8.153310045599937e-05, 7.687210745643824e-05, 7.28695304132998e-05, 7.002452184678987e-05, 6.713537004543468e-05, 6.28554989816621e-05, 5.9997179050697014e-05, 5.8569276006892323e-05, 5.589217835222371e-05, 5.439653614303097e-05, 5.1458839152473956e-05, 4.9563539505470544e-05, 4.9400834541302174e-05, 4.788253500009887e-05, 4.58069916930981e-05, 4.4847241952084005e-05, 4.315644400776364e-05, 4.130572051508352e-05, 3.972699778387323e-05, 3.8446669350378215e-05, 3.688568904181011e-05, 3.635714529082179e-05, 3.5175849916413426e-05, 3.477348946034908e-05, 3.3142448955914006e-05, 3.2293501135427505e-05, 3.13731943606399e-05, 3.1002669857116416e-05, 2.9731392714893445e-05, 2.9111002731951885e-05, 2.905324254243169e-05, 2.8808244678657502e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 0.00017190765356644988, "first_to_last_ratio": 1.4894621368061003, "imbalance_abs_log_ratio": 0.3984150727783497, "last_quartile_mean": 0.00011541592721187044, "mean": 0.00013894661265112518, "normalized": [ 1.3739040544627474, 1.3296102750467884, 1.2883342981472405, 1.2499918159027226, 1.2130627732820631, 1.175544413473833, 1.1474083142393565, 1.1199112790671786, 1.092909618037032, 1.0755929927697745, 1.0512861461262244, 1.0312571117882559, 1.0194388253067554, 1.0019061415077835, 0.980344379776871, 0.9647961249933756, 0.9493075664595542, 0.9344817410273759, 0.9198929201794295, 0.9079992279088673, 0.8952920956888009, 0.8869530532606651, 0.8771902552778059, 0.8683889858357474, 0.8579407806493952, 0.8482823463732029, 0.8408658204630546, 0.8330843095877946, 0.8243382853592501, 0.818115422616924, 0.8135611224181157, 0.8090075029660142 ], "peak_layer": 1, "peak_normalized": 1.3739040544627474, "population_cv": 0.16161403917150835, "rest_mean": 0.00014210757001786045, "spike_contrast": 0.8576421569620596, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012187744287075474, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00019089931447524577, 0.00018474484386388212, 0.00017900968668982387, 0.00017368212866131216, 0.0001685509632807225, 0.00016333791427314281, 0.00015942849859129637, 0.00015560787869617343, 0.00015185608936008066, 0.00014945000293664634, 0.00014607264893129468, 0.0001432896824553609, 0.00014164757158141583, 0.0001392114645568654, 0.00013621553080156446, 0.0001340551534667611, 0.00013190307072363794, 0.00012984307250007987, 0.0001278160052606836, 0.00012616341700777411, 0.0001243978040292859, 0.00012323912233114243, 0.00012188261462142691, 0.00012065970804542303, 0.00011920796532649547, 0.00011786595860030502, 0.00011683545744745061, 0.00011575424287002534, 0.00011453901242930442, 0.00011367436673026532, 0.00011304156214464456, 0.00011240885214647278 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 0.00014552358334185556, "first_to_last_ratio": 1.262638843036328, "imbalance_abs_log_ratio": 0.2332038508028458, "last_quartile_mean": 0.00011525352965691127, "mean": 0.00012953498708156985, "normalized": [ 1.1676304712442283, 1.1525836984023852, 1.1391521450612143, 1.1288261090295841, 1.1167600441451297, 1.1054423860149858, 1.094731137320412, 1.082319740349084, 1.067957453000188, 1.0563935463552665, 1.04560972624621, 1.0337959762176818, 1.0235535208863231, 1.0159040893770892, 1.0050628637064216, 0.9950792389307415, 0.9865814183778941, 0.9786230528286792, 0.9704418054120744, 0.9608381133357455, 0.9508407831226554, 0.9429749854457838, 0.9346839821902475, 0.9262276149685831, 0.9170542960948607, 0.9059488475652709, 0.8984648923618161, 0.8908037775288885, 0.8831483920181569, 0.8778579808382188, 0.8744051093280881, 0.8703028022960919 ], "peak_layer": 1, "peak_normalized": 1.1676304712442283, "population_cv": 0.0896642712448568, "rest_mean": 0.00013110964479252557, "spike_contrast": 0.9231344927669946, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012103183544240892, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00015124899800866842, 0.00014929991448298097, 0.00014756005839444697, 0.00014622247545048594, 0.00014465949789155275, 0.00014319346519187093, 0.00014180598373059183, 0.00014019827358424664, 0.00013833785487804562, 0.00013683992438018322, 0.0001354430423816666, 0.00013391274842433631, 0.0001325859921053052, 0.00013159512309357524, 0.00013019080506637692, 0.00012889757636003196, 0.00012779681128449738, 0.00012676592450588942, 0.00012570616672746837, 0.00012446215259842575, 0.00012316714855842292, 0.00012214825255796313, 0.00012107427755836397, 0.00011997888213954866, 0.00011879061639774591, 0.00011735207226593047, 0.0001163826382253319, 0.0001153902558144182, 0.00011439861555118114, 0.00011371332220733166, 0.00011326605454087257, 0.0001127346622524783 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00029742828155576717, "first_to_last_ratio": 2.2781890121807993, "imbalance_abs_log_ratio": 0.8233808344454435, "last_quartile_mean": 0.00013055469935352448, "mean": 0.00019329176166138495, "normalized": [ 1.8891850069388627, 1.7801603926271052, 1.6773544463608039, 1.566500262900107, 1.476456430986117, 1.3754925511314788, 1.3084504582544147, 1.236424592515421, 1.172965600373877, 1.150871569148191, 1.0802294364746856, 1.0319209662753672, 1.0080063247818516, 0.9698985570693867, 0.9067931019504507, 0.8756163462587222, 0.8444435806568995, 0.8113939671049883, 0.7809677247391495, 0.7613442367276676, 0.7402582678385915, 0.7293562882295931, 0.7162426695419845, 0.7062419232587185, 0.6940340549797547, 0.6866100033591334, 0.6831688143117768, 0.6753320518823956, 0.6715919825156708, 0.6686410475337363, 0.6640239865195754, 0.6600233567535221 ], "peak_layer": 1, "peak_normalized": 1.8891850069388627, "population_cv": 0.357915029863326, "rest_mean": 0.00020341356920888993, "spike_contrast": 0.6815376252628051, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001386340009048581, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00036516389809548855, 0.0003440903383307159, 0.0003242187958676368, 0.0003027915954589844, 0.00028538686456158757, 0.00026587137836031616, 0.0002529126941226423, 0.00023899068764876574, 0.00022672458726447076, 0.00022245399304665625, 0.0002087994507746771, 0.00019946182146668434, 0.00019483931828290224, 0.00018747340072877705, 0.00017527563613839447, 0.00016924942610785365, 0.00016322398732881993, 0.00015683576930314302, 0.0001509546273155138, 0.00014716156874783337, 0.0001430858246749267, 0.0001409785618307069, 0.00013844380737282336, 0.00013651074550580233, 0.00013415106514003128, 0.00013271605712361634, 0.00013205090363044292, 0.00013053612201474607, 0.00012981319741811603, 0.00012924280599690974, 0.00012835036613978446, 0.00012757707736454904 ] } } }, "pre_attention_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "de71d4cf2d509a1063f6eac7f50c4885e9a058dee7e4b993ad40f5527f528d00", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.02802751213312149, 0.01413465291261673, 0.014225807972252369, 0.01433455292135477, 0.01440478302538395, 0.009703479707241058, 0.00977881159633398, 0.009858808480203152, 0.009950309991836548, 0.00752287870272994, 0.007571155205368996, 0.007693085819482803, 0.007788818795233965, 0.006353048607707024, 0.006389216054230928, 0.0064707002602517605, 0.00656962301582098, 0.005553483963012695, 0.0056249531917274, 0.005688921548426151, 0.005768724251538515, 0.004966245498508215, 0.005057469941675663, 0.005098348017781973, 0.005216081161051989, 0.0046758754178881645, 0.004727107938379049, 0.004777040798217058, 0.004837613552808762, 0.0044186655431985855, 0.004510571248829365, 0.0045849340967834 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.0001580166108396952, "first_to_last_ratio": 1.9713740353643698, "imbalance_abs_log_ratio": 0.6787307795042399, "last_quartile_mean": 8.015557068574708e-05, "mean": 0.00011843875881822896, "normalized": [ 0.8918679597819287, 1.5814819593588525, 1.5722239987443252, 1.3246067564285042, 1.1252650527356378, 1.512743589128775, 1.3788112428745087, 1.2863035124436846, 1.097095055843533, 1.468154353173571, 1.1523254606068964, 1.0839888571909224, 1.054518217028451, 1.146335080549274, 1.1665208527784754, 0.8931136212023528, 0.8203804970547371, 1.073741223277434, 0.919654742394771, 0.8223971330190447, 0.7711288002244273, 0.8470775332584999, 0.8298711629738247, 0.7662487463744663, 0.7210382462310316, 0.7071376059209309, 0.7189559392079661, 0.727980518763353, 0.6111916361855597, 0.6986829946379066, 0.6244678178312321, 0.6046898327751223 ], "peak_layer": 2, "peak_normalized": 1.5814819593588525, "population_cv": 0.2904715201464359, "rest_mean": 0.00012310891098523926, "spike_contrast": 0.7572151875143335, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.321993711637333e-05, "top_five_layers": [ 2, 3, 6, 10, 7 ], "top_five_spike_overlap": 0 }, "values": [ 0.00010563173418631777, 0.0001873087603598833, 0.00018621225899551064, 0.00015688478015363216, 0.0001332749961875379, 0.00017916747310664505, 0.00016330469225067645, 0.00015234819147735834, 0.00012993857671972364, 0.00017388637934345752, 0.0001364799973089248, 0.0001283862948184833, 0.00012489582877606153, 0.00013577050413005054, 0.00013816128193866462, 0.00010577926877886057, 9.71648478298448e-05, 0.00012717257777694613, 0.00010892276623053476, 9.74036956904456e-05, 9.133153798757121e-05, 0.0001003268116619438, 9.828891052166e-05, 9.07535504666157e-05, 8.539887494407594e-05, 8.375250035896897e-05, 8.515224908478558e-05, 8.622110908618197e-05, 7.238877878990024e-05, 8.275114669231698e-05, 7.396119326585904e-05, 7.161871326388791e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.0001664285200604354, "first_to_last_ratio": 1.9719337364651075, "imbalance_abs_log_ratio": 0.6790146534176558, "last_quartile_mean": 8.439863722742302e-05, "mean": 0.0001245867656507471, "normalized": [ 0.8984334461398115, 1.5927952483854981, 1.5658595485525297, 1.3198735865176374, 1.131213894321221, 1.5125315092450449, 1.3796810696180641, 1.286366052942558, 1.0960606282761096, 1.4536884557404346, 1.1543342746982923, 1.0797559630746076, 1.0497530570909623, 1.1457182987306291, 1.1551962694199982, 0.9023721663305979, 0.8238710779144349, 1.0632503993742801, 0.9211624832492573, 0.8274335806257342, 0.7739032412019898, 0.850990856837025, 0.8281317613098801, 0.7681943949939335, 0.7208251570035091, 0.7089546837555952, 0.7203162530736348, 0.7236174125342815, 0.6145740290254166, 0.6994991973458914, 0.6274381906330746, 0.6042038120380651 ], "peak_layer": 2, "peak_normalized": 1.5927952483854981, "population_cv": 0.2889665774838781, "rest_mean": 0.00012946851159169995, "spike_contrast": 0.7586812913967177, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.822533756960183e-05, "top_five_layers": [ 2, 3, 6, 10, 7 ], "top_five_spike_overlap": 0 }, "values": [ 0.00011193291720701382, 0.0001984412083402276, 0.0001950853766174987, 0.000164438781212084, 0.00014093428035266697, 0.00018844140868168324, 0.00017189000209327787, 0.00016026418597903103, 0.00013655464863404632, 0.00018111034296453, 0.00014381477376446128, 0.0001345233031315729, 0.00013078533811494708, 0.00014274133718572557, 0.00014392216689884663, 0.00011242362961638719, 0.00010264343291055411, 0.0001324669283349067, 0.00011476465442683548, 0.00010308727360097691, 9.641810174798593e-05, 0.00010602219845168293, 0.00010317425767425448, 9.570685506332666e-05, 8.980527491075918e-05, 8.832637104205787e-05, 8.974187221610919e-05, 9.015315299620852e-05, 7.656779052922502e-05, 8.714834257261828e-05, 7.817049481673166e-05, 7.527579873567447e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.14759047608822584, "first_to_last_ratio": 1.9719337947467823, "imbalance_abs_log_ratio": 0.6790146829732503, "last_quartile_mean": 0.07484555337578058, "mean": 0.1104847895912826, "normalized": [ 0.8984334974418968, 1.5927952138130326, 1.565859646322909, 1.3198735575362357, 1.1312139236939245, 1.5125315007911495, 1.3796810264894532, 1.286366010451999, 1.0960606168617568, 1.4536884922319049, 1.1543342013148132, 1.0797558953237962, 1.0497530983161607, 1.1457182562234747, 1.1551962948455465, 0.9023721940039697, 0.8238711061937815, 1.0632504861804335, 0.9211625151546727, 0.8274335809310764, 0.7739032642161865, 0.8509909070635955, 0.8281317391228538, 0.7681943896830757, 0.7208251689669645, 0.7089546589522648, 0.7203162343708721, 0.7236173971219663, 0.6145740294939214, 0.6994991437375198, 0.6274381319743458, 0.6042038211744466 ], "peak_layer": 2, "peak_normalized": 1.5927952138130326, "population_cv": 0.28896658018674404, "rest_mean": 0.11481397047086998, "spike_contrast": 0.7586813040631772, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.08710721284151077, "top_five_layers": [ 2, 3, 6, 10, 7 ], "top_five_spike_overlap": 0 }, "values": [ 0.09926323592662811, 0.1759796440601349, 0.1730036735534668, 0.14582595229148865, 0.12498193234205246, 0.16711172461509705, 0.15243376791477203, 0.14212387800216675, 0.12109802663326263, 0.16061046719551086, 0.1275363713502884, 0.1192966029047966, 0.11598175019025803, 0.12658444046974182, 0.12763161957263947, 0.09969840198755264, 0.0910252258181572, 0.11747300624847412, 0.10177444666624069, 0.09141882508993149, 0.08550453931093216, 0.09402155131101608, 0.09149596095085144, 0.08487379550933838, 0.0796402171254158, 0.0783287063241005, 0.07958398759365082, 0.07994871586561203, 0.06790108233690262, 0.07728401571512222, 0.069322369992733, 0.06675533205270767 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 5.093040158499207e-05, "first_to_last_ratio": 0.8860093039525244, "imbalance_abs_log_ratio": 0.12102782735710801, "last_quartile_mean": 5.748291960117058e-05, "mean": 5.7093861016710434e-05, "normalized": [ 0.4337136137178416, 0.9185569667519601, 0.9444095926066538, 0.8728787927927439, 0.7849140301431445, 1.1296292591692207, 1.0726593944235638, 0.9796135302182263, 0.9386642177229508, 1.3046353648934683, 1.0461832118506482, 1.0032917361864164, 0.9662905330123907, 1.0760080966276753, 1.1828739682986207, 0.9087955578115051, 0.9194941496879511, 1.198292373503509, 1.0946132296674007, 1.0673695485587213, 0.9244260207272295, 1.0525679458300632, 1.125653431968099, 0.9999504891362015, 0.9629027708994021, 1.0063234987710272, 0.9975933861195996, 1.0791525148711678, 0.9412468232151083, 1.085173475571988, 1.0031556955576197, 0.9789667796878816 ], "peak_layer": 10, "peak_normalized": 1.3046353648934683, "population_cv": 0.1441687540611157, "rest_mean": 5.695535414647397e-05, "spike_contrast": 1.0155638391297448, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.784179811598733e-05, "top_five_layers": [ 10, 18, 15, 6, 23 ], "top_five_spike_overlap": 1 }, "values": [ 2.4762384782661684e-05, 5.2443963795667514e-05, 5.3919990023132414e-05, 4.9836020480142906e-05, 4.481377254705876e-05, 6.449489592341706e-05, 6.124226638348773e-05, 5.592991874436848e-05, 5.3591964388033375e-05, 7.448667020071298e-05, 5.973063889541663e-05, 5.728179894504137e-05, 5.516925739357248e-05, 6.143345672171563e-05, 6.75348419463262e-05, 5.1886647270293906e-05, 5.249747118796222e-05, 6.84151382301934e-05, 6.249569560168311e-05, 6.094024865888059e-05, 5.277905074763112e-05, 6.009516800986603e-05, 6.426790059776977e-05, 5.7091034250333905e-05, 5.4975836974335834e-05, 5.74548939766828e-05, 5.695645813830197e-05, 6.161298369988799e-05, 5.373941530706361e-05, 6.19567435933277e-05, 5.727403186028823e-05, 5.58929932594765e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 5.801741053801379e-05, "first_to_last_ratio": 0.954195857595535, "imbalance_abs_log_ratio": 0.046886327142486896, "last_quartile_mean": 6.08024129178375e-05, "mean": 6.296396202287724e-05, "normalized": [ 0.4796364558374551, 0.9568353419130547, 0.9804638447804884, 0.8949545354119591, 0.8237377234766136, 1.1597846916175047, 1.085649359633547, 0.9904450076993521, 0.9595078978373608, 1.2941748206155441, 1.0680753760655664, 1.0231559828609242, 0.9904019047543832, 1.096490963646882, 1.1850351594596897, 0.9422215946487361, 0.9418631353047864, 1.1705024179981143, 1.0926353878857977, 1.0382275689133567, 0.9320833313178482, 1.0784583323883348, 1.0985080196909318, 0.9917906747078287, 0.9476388143731139, 0.9855066582268529, 0.978396290111904, 1.028407839812513, 0.8982055143685478, 1.0282804954549911, 0.9451124964564701, 0.9138123627295472 ], "peak_layer": 10, "peak_normalized": 1.2941748206155441, "population_cv": 0.13431233496292436, "rest_mean": 6.285090866084729e-05, "spike_contrast": 1.0115120295379672, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.357445017783903e-05, "top_five_layers": [ 10, 15, 18, 6, 23 ], "top_five_spike_overlap": 1 }, "values": [ 3.019981159013696e-05, 6.0246144130360335e-05, 6.173388828756288e-05, 5.634988337988034e-05, 5.186579073779285e-05, 7.302463927771896e-05, 6.835678505012766e-05, 6.236234185053036e-05, 6.041441884008236e-05, 8.148637425620109e-05, 6.725025741616264e-05, 6.442195444833487e-05, 6.235962791834027e-05, 6.903941539349034e-05, 7.461450877599418e-05, 5.9326004702597857e-05, 5.930343468207866e-05, 7.369946979451925e-05, 6.879665306769311e-05, 6.537092122016475e-05, 5.8687659475253895e-05, 6.790400948375463e-05, 6.916641723364592e-05, 6.244707037694752e-05, 5.966709431959316e-05, 6.205140380188823e-05, 6.160370685392991e-05, 6.475263216998428e-05, 5.655457789544016e-05, 6.474461406469345e-05, 5.950802733423188e-05, 5.753724690293893e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 1.7888660295284353e-05, "first_to_last_ratio": 0.41855462748958633, "imbalance_abs_log_ratio": 0.8709478659353102, "last_quartile_mean": 4.2739129185065394e-05, "mean": 3.1986895351110434e-05, "normalized": [ 0.2593955501951102, 0.4997829486650695, 0.48480994473028793, 0.4968760363401999, 0.5474125689219601, 0.7788406177150305, 0.7554473604850604, 0.6514322393717875, 0.7578135841604356, 0.9674810188795085, 0.9032772556875033, 0.9019220651946442, 0.8311356963419073, 0.9587970177975769, 1.1401394869948454, 0.8953886492354074, 1.0262272842074007, 1.1413916918814533, 1.174037958462901, 1.194927715786966, 1.0267140632187253, 1.3434898267078448, 1.3729067467361273, 1.2011929483563188, 1.1735959904166378, 1.3266745689353452, 1.2739952988364465, 1.4049535244931144, 1.2752889026856997, 1.4680979746684197, 1.4182292846928506, 1.3483241791974143 ], "peak_layer": 30, "peak_normalized": 1.4680979746684197, "population_cv": 0.3184955307507873, "rest_mean": 3.066251991657299e-05, "spike_contrast": 1.276428773763581, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.913852269761264e-05, "top_five_layers": [ 30, 31, 28, 23, 32 ], "top_five_spike_overlap": 1 }, "values": [ 8.297258318634704e-06, 1.5986504877218977e-05, 1.5507564967265353e-05, 1.589352177688852e-05, 1.7510028555989265e-05, 2.491269333404489e-05, 2.416441566310823e-05, 2.083729486912489e-05, 2.4240103812189773e-05, 3.094671410508454e-05, 2.889303505071439e-05, 2.8849686714238487e-05, 2.6585450541460887e-05, 3.066893987124786e-05, 3.6469522456172854e-05, 2.8640703021665104e-05, 3.282582474639639e-05, 3.6509576602838933e-05, 3.755382931558415e-05, 3.822202779701911e-05, 3.2841395295690745e-05, 4.2974068492185324e-05, 4.391502443468198e-05, 3.842243313556537e-05, 3.7539692129939795e-05, 4.243620060151443e-05, 4.0751154301688075e-05, 4.494010136113502e-05, 4.0792532672639936e-05, 4.695989628089592e-05, 4.5364751713350415e-05, 4.3128704419359565e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00022091567916504573, "first_to_last_ratio": 1.232986382535498, "imbalance_abs_log_ratio": 0.20943917994887448, "last_quartile_mean": 0.00017917122386279516, "mean": 0.0002198539991695725, "normalized": [ 0.4672441326905094, 1.0299890566632133, 1.08986441571188, 1.0084939820045402, 0.8814237539069011, 1.2693826047962031, 1.1886514905905403, 1.1035827467935222, 1.0473307057033197, 1.4274044616309502, 1.1442998336482333, 1.0911283607583675, 1.1094038041384326, 1.1946873974128753, 1.2511466100570947, 0.9923165311956169, 0.9239703707694655, 1.2473578196113966, 1.1181336030529359, 1.0160745414007064, 0.9397394354798955, 0.9905446518023159, 1.0121913654482901, 0.9359943959598221, 0.8839446941554127, 0.8510229499852975, 0.8648513533654478, 0.8786898836618169, 0.7455559493639876, 0.8399702482950041, 0.7392957942269532, 0.7163130557190521 ], "peak_layer": 10, "peak_normalized": 1.4274044616309502, "population_cv": 0.18952709251819305, "rest_mean": 0.0002217885959436427, "spike_contrast": 0.9441746799407325, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00020940717658959329, "top_five_layers": [ 10, 6, 15, 18, 14 ], "top_five_spike_overlap": 0 }, "values": [ 0.00010272549116052687, 0.00022644721320830286, 0.00023961105034686625, 0.00022172143508214504, 0.0001937845372594893, 0.0002790788421407342, 0.0002613297838252038, 0.00024262708029709756, 0.00023025984410196543, 0.000313820579322055, 0.00025157889467664063, 0.00023988893372006714, 0.00024390686303377151, 0.000262656802078709, 0.00027506958576850593, 0.0002181647578254342, 0.00020313858112785965, 0.00027423660503700376, 0.0002458261442370713, 0.00022338805138133466, 0.00020660547306761146, 0.00021777520305477083, 0.00022253431961871684, 0.00020578211115207523, 0.0001943387760547921, 0.00018710079893935472, 0.0001901410287246108, 0.00019318348495289683, 0.00016391345707233995, 0.00018467081827111542, 0.00016253713693004102, 0.00015748428995721042 ] } } }, "pre_mlp_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "2999a257265024350fd2665e2a8d452a03cd662193b4245d15290408ad29922d", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.014061377383768559, 0.014165290631353855, 0.014281988143920898, 0.014350161887705326, 0.00966146495193243, 0.009741888381540775, 0.009820535778999329, 0.00991273857653141, 0.0074927168898284435, 0.007546746172010899, 0.00766373984515667, 0.007762390188872814, 0.006325984839349985, 0.006375366821885109, 0.006445344537496567, 0.006555322092026472, 0.005531130358576775, 0.005611324217170477, 0.0056777093559503555, 0.00574630219489336, 0.004946626722812653, 0.0050392793491482735, 0.005084683652967215, 0.005203457549214363, 0.004667261149734259, 0.004717131145298481, 0.004768412560224533, 0.004826710093766451, 0.0044052014127373695, 0.004504534415900707, 0.004574235528707504, 0.004644096828997135 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 3.729895206561196e-05, "first_to_last_ratio": 1.2469721805352876, "imbalance_abs_log_ratio": 0.22071835733540474, "last_quartile_mean": 2.9911615229138988e-05, "mean": 3.340667609563752e-05, "normalized": [ 1.308752778077389, 1.2456464596610868, 1.0236956931411572, 0.9922285647386262, 1.26903181062375, 1.1170846373362797, 1.0287316533628064, 0.9469237624012169, 1.1300261751959062, 1.1132698789554223, 1.0126240729566007, 0.9379202574439188, 1.1130503370514597, 1.070713488608195, 0.9897642722064988, 0.922085688720865, 1.036831181462568, 0.9898105546019274, 0.9395130252451869, 0.9001967292772246, 0.9975952535130206, 0.9404117749145336, 0.9224862220158919, 0.8885787590271026, 0.9461670180110204, 0.8974372095121397, 0.8904691498559152, 0.8684437934704528, 0.9244147277585869, 0.9015689206268677, 0.8740901912628669, 0.8604359589635168 ], "peak_layer": 1, "peak_normalized": 1.308752778077389, "population_cv": 0.1164905978879517, "rest_mean": 3.378375206213807e-05, "spike_contrast": 0.9285666618330386, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.137046587653458e-05, "top_five_layers": [ 1, 5, 2, 9, 6 ], "top_five_spike_overlap": 0 }, "values": [ 4.3721080146497115e-05, 4.161290780757554e-05, 3.419827044126578e-05, 3.314705827506259e-05, 4.239413465256803e-05, 3.73180846509058e-05, 3.4366505133220926e-05, 3.1633575417799875e-05, 3.7750418414361775e-05, 3.719064625329338e-05, 3.3828404411906376e-05, 3.1332798243965954e-05, 3.718331208801828e-05, 3.5768978705164045e-05, 3.306473445263691e-05, 3.080381793552078e-05, 3.463708344497718e-05, 3.3066280593629926e-05, 3.138600732199848e-05, 3.007258055731654e-05, 3.332634150865488e-05, 3.1416031561093405e-05, 3.0817198421573266e-05, 2.9684462788281962e-05, 3.1608295103069395e-05, 2.9980394174344838e-05, 2.9747614462394267e-05, 2.9011820515734144e-05, 3.0881623388268054e-05, 3.01184209092753e-05, 2.9200447897892445e-05, 2.874430538213346e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 6.705871965095866e-05, "first_to_last_ratio": 0.7180297289018083, "imbalance_abs_log_ratio": 0.33124430563536084, "last_quartile_mean": 9.33926785364747e-05, "mean": 8.228966771639534e-05, "normalized": [ 0.8685826408392191, 0.8249006353131848, 0.7154905339803773, 0.6884730939760333, 0.9470487095509537, 0.8638069620260335, 0.8256155016985178, 0.7853664463613442, 0.9851686390305121, 0.9692511243547932, 0.9083804947955202, 0.8729855457896263, 1.0422548534804916, 1.0255492616133721, 0.9871528463964067, 0.9398223255303129, 1.0958375595009802, 1.0726325633807163, 1.0495259774381736, 1.0230838788771643, 1.1576441025653552, 1.1142532603054807, 1.1008567438213033, 1.056908774400763, 1.159063578749664, 1.1192434432436902, 1.1032241584740294, 1.1060435702772593, 1.1786235078649367, 1.1628365878275773, 1.141840825041133, 1.1085318534950752 ], "peak_layer": 29, "peak_normalized": 1.1786235078649367, "population_cv": 0.1341844256762087, "rest_mean": 8.049536754023629e-05, "spike_contrast": 1.1426606459269575, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.197888866765424e-05, "top_five_layers": [ 29, 30, 25, 21, 31 ], "top_five_spike_overlap": 2 }, "values": [ 7.14753768988885e-05, 6.788079917896539e-05, 5.887747829547152e-05, 5.66542221349664e-05, 7.793232362018898e-05, 7.108238787623122e-05, 6.793962529627606e-05, 6.462754390668124e-05, 8.106919995043427e-05, 7.975935295689851e-05, 7.475032907677814e-05, 7.183769048424438e-05, 8.576680556870997e-05, 8.4392107964959e-05, 8.123247971525416e-05, 7.733766688033938e-05, 9.017610864248127e-05, 8.826657722238451e-05, 8.636514394311234e-05, 8.418923243880272e-05, 9.526214853394777e-05, 9.169153054244816e-05, 9.0589135652408e-05, 8.697267185198143e-05, 9.537895675748587e-05, 9.210217103827745e-05, 9.078394941752777e-05, 9.101595787797123e-05, 9.698853682493791e-05, 9.56894364207983e-05, 9.396170207764953e-05, 9.122071787714958e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.05946834245696664, "first_to_last_ratio": 0.7180297222365345, "imbalance_abs_log_ratio": 0.33124431491808765, "last_quartile_mean": 0.08282156102359295, "mean": 0.07297530013602227, "normalized": [ 0.8685826122359729, 0.8249005917343738, 0.71549052911611, 0.6884731350635278, 0.9470486699630107, 0.8638070113577097, 0.8256154769796886, 0.7853664269052657, 0.9851686341386658, 0.9692511556795396, 0.9083805454214465, 0.8729855579949275, 1.0422548040385637, 1.0255492365750694, 0.9871528950273681, 0.9398223256645052, 1.0958375053304956, 1.07263252629438, 1.0495259690486791, 1.0230837911446407, 1.1576441435704576, 1.1142533045483842, 1.1008568124470575, 1.0569088284972943, 1.1590636022344982, 1.119243411839159, 1.1032241423789504, 1.1060435591240758, 1.1786235037173105, 1.1628366076958818, 1.141840809884862, 1.1085318743481278 ], "peak_layer": 29, "peak_normalized": 1.1786235037173105, "population_cv": 0.13418442668798974, "rest_mean": 0.07138409617322462, "spike_contrast": 1.1426607032635474, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.08156780153512955, "top_five_layers": [ 29, 30, 25, 21, 31 ], "top_five_spike_overlap": 2 }, "values": [ 0.06338507682085037, 0.0601973682641983, 0.05221313610672951, 0.050241533666849136, 0.0691111609339714, 0.06303657591342926, 0.060249537229537964, 0.05731235072016716, 0.07189297676086426, 0.07073139399290085, 0.06628934293985367, 0.06370638310909271, 0.07605885714292526, 0.0748397633433342, 0.07203777879476547, 0.06858381628990173, 0.07996907085180283, 0.07827568054199219, 0.07658947259187698, 0.07465984672307968, 0.0844794288277626, 0.08131296932697296, 0.0803353562951088, 0.07712823897600174, 0.08458301424980164, 0.08167712390422821, 0.08050811290740967, 0.08071386069059372, 0.0860104039311409, 0.08485835045576096, 0.08332617580890656, 0.08089544624090195 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 7.408341275549901e-06, "first_to_last_ratio": 0.49124437919492936, "imbalance_abs_log_ratio": 0.7108135577065074, "last_quartile_mean": 1.5080765479069669e-05, "mean": 1.1844782648040564e-05, "normalized": [ 0.5657990507718705, 0.5382233871697658, 0.49942145823009426, 0.48906984415931193, 0.8062921756393574, 0.7122953827323945, 0.7013508394120195, 0.6911626993599347, 0.8940675881372303, 0.868722127857376, 0.9735911331661544, 0.854855860776803, 0.989483587379368, 1.0059088486455374, 1.0303729006301217, 0.9941025538400318, 1.1620912657632405, 1.127012997009919, 1.113401270263767, 1.066097770938846, 1.1826034542812638, 1.1643547935214302, 1.2049778184223279, 1.1791491538213539, 1.2365172470752577, 1.219921447624247, 1.2337761202582145, 1.2288278254433027, 1.3039217543779833, 1.3631114810840155, 1.2856635723507386, 1.3138525898567213 ], "peak_layer": 30, "peak_normalized": 1.3631114810840155, "population_cv": 0.2549101540724733, "rest_mean": 1.1420299651271543e-05, "spike_contrast": 1.237882653019464, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.4136990830593277e-05, "top_five_layers": [ 30, 32, 29, 31, 25 ], "top_five_spike_overlap": 1 }, "values": [ 6.701766778860474e-06, 6.3751390371180605e-06, 5.915538622502936e-06, 5.7929260037781205e-06, 9.550355571263935e-06, 8.436983989668079e-06, 8.307348252856173e-06, 8.186671948351432e-06, 1.0590036254143342e-05, 1.0289824786013924e-05, 1.1531975360412616e-05, 1.0125581866304856e-05, 1.1720218026312068e-05, 1.1914771675947122e-05, 1.220454305439489e-05, 1.1774928680097219e-05, 1.3764718460151926e-05, 1.334922399109928e-05, 1.3187996046326589e-05, 1.2627696378331166e-05, 1.4007680874783546e-05, 1.379152945446549e-05, 1.4272700354922563e-05, 1.3966765436634887e-05, 1.4646278032159898e-05, 1.4449704394792207e-05, 1.4613809980801307e-05, 1.4555198504240252e-05, 1.5444669770658948e-05, 1.614575921848882e-05, 1.5228405572997872e-05, 1.5562298358418047e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 5.6029601637419546e-05, "first_to_last_ratio": 0.6070213107493427, "imbalance_abs_log_ratio": 0.49919138022039955, "last_quartile_mean": 9.230252817360451e-05, "mean": 7.840498903988191e-05, "normalized": [ 0.6598133090619259, 0.6419677920829977, 0.6005256373772865, 0.5913463573822231, 0.8562376513989928, 0.8062968559250673, 0.7923156566726156, 0.768439227921366, 0.9790122729268586, 0.9665730330981341, 0.9163710065350301, 0.8881686280668529, 1.0631190324760253, 1.0511030519715439, 1.0188080221730096, 0.9712812246823179, 1.1361272291219966, 1.1138561419734825, 1.0918134348286874, 1.0639217496772315, 1.203414696914147, 1.1592426077267617, 1.1442664103435203, 1.097952968622282, 1.2042281788778202, 1.1625445130141736, 1.1452986211445626, 1.1473932954411665, 1.2233942816523278, 1.205318760675343, 1.1829469858778916, 1.146901364356358 ], "peak_layer": 29, "peak_normalized": 1.2233942816523278, "population_cv": 0.1898597249088972, "rest_mean": 7.605543874820728e-05, "spike_contrast": 1.1977126437531995, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.109256061492487e-05, "top_five_layers": [ 29, 30, 25, 21, 31 ], "top_five_spike_overlap": 2 }, "values": [ 5.173265526536852e-05, 5.033347770222463e-05, 4.708420601673424e-05, 4.636450466932729e-05, 6.713330367347226e-05, 6.321769615169615e-05, 6.212150037754327e-05, 6.024946924299002e-05, 7.675944652874023e-05, 7.578414806630462e-05, 7.184805872384459e-05, 6.963685154914856e-05, 8.335383608937263e-05, 8.241172326961532e-05, 7.987963181221858e-05, 7.615329377586022e-05, 8.907804294722155e-05, 8.733187860343605e-05, 8.560362039133906e-05, 8.341677312273532e-05, 9.435371612198651e-05, 9.089040395338088e-05, 8.971619536168873e-05, 8.608499047113582e-05, 9.441749716643244e-05, 9.114928980125114e-05, 8.97971258382313e-05, 8.996135875349864e-05, 9.592021524440497e-05, 9.450300422031432e-05, 9.274894546251744e-05, 8.992278890218586e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 4.8511469685763586e-05, "first_to_last_ratio": 0.5271352332510719, "imbalance_abs_log_ratio": 0.6402981537748061, "last_quartile_mean": 9.202850924339145e-05, "mean": 7.532146378252946e-05, "normalized": [ 0.548104082267993, 0.5416830199005969, 0.528884702689083, 0.5247448675895723, 0.7777813723908718, 0.7487613756360456, 0.7496059866168319, 0.7329069161583202, 0.9480676391902623, 0.9399123881532756, 0.9000936195631334, 0.883441930780847, 1.0560589316963704, 1.0534552095740957, 1.032095781768693, 0.9835620691098637, 1.1601754560133901, 1.1467788545782247, 1.1233340534265408, 1.0967375255670042, 1.2396083970871608, 1.1951467072205837, 1.1828066060767042, 1.1317734013234224, 1.244698956870482, 1.2025862582796845, 1.1881970083126314, 1.1882256015362047, 1.2689460104605979, 1.2589162611011824, 1.2307315535498626, 1.1921774555104672 ], "peak_layer": 29, "peak_normalized": 1.2689460104605979, "population_cv": 0.23486334953575583, "rest_mean": 7.254842300123225e-05, "spike_contrast": 1.244629176846487, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.029588400153444e-05, "top_five_layers": [ 29, 30, 25, 21, 31 ], "top_five_spike_overlap": 2 }, "values": [ 4.1284001781605184e-05, 4.080035796505399e-05, 3.9836369978729635e-05, 3.952455153921619e-05, 5.858363147126511e-05, 5.639780283672735e-05, 5.6461420172126964e-05, 5.520362174138427e-05, 7.140984234865755e-05, 7.079557690303773e-05, 6.77963689668104e-05, 6.654213939327747e-05, 7.954390457598493e-05, 7.934778841445222e-05, 7.773896504659206e-05, 7.408333476632833e-05, 8.738611359149218e-05, 8.637706196168438e-05, 8.461116522084922e-05, 8.26078758109361e-05, 9.336911898571998e-05, 9.002019942272455e-05, 8.909072494134307e-05, 8.524682925781235e-05, 9.375254740007222e-05, 9.058055729838088e-05, 8.949673792812973e-05, 8.949889161158353e-05, 9.557887096889317e-05, 9.482341556577012e-05, 9.270050213672221e-05, 8.979655103757977e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 9.61669284151867e-05, "first_to_last_ratio": 0.8297833624837898, "imbalance_abs_log_ratio": 0.18659062131596024, "last_quartile_mean": 0.00011589401856326731, "mean": 0.00010716335873439675, "normalized": [ 0.9380742209455786, 0.8898820858744699, 0.7837417114644826, 0.7583102664636657, 1.0822070619439073, 0.9520749081713696, 0.9107982467806989, 0.8640017807748708, 1.0669538303069324, 1.0266950649381863, 0.9594029855154438, 0.8901506143407661, 1.0795289063552718, 1.0379165674077444, 0.9706971366471856, 0.914979822300863, 1.052939971267624, 1.0225707935964177, 0.9851238030379684, 0.9636652893153618, 1.0990044510799908, 1.0523904893769196, 1.0422234788441622, 1.0049019599938585, 1.0962247910521359, 1.0599378706264646, 1.05039930496957, 1.0583562345651827, 1.1199788667078585, 1.1100529546792268, 1.0850973923636869, 1.0717171382921347 ], "peak_layer": 29, "peak_normalized": 1.1199788667078585, "population_cv": 0.09156717959150794, "rest_mean": 0.00010599351123716958, "spike_contrast": 1.0706366257223145, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00011348053521942347, "top_five_layers": [ 29, 30, 21, 25, 31 ], "top_five_spike_overlap": 2 }, "values": [ 0.00010052718425868079, 9.536275319987908e-05, 8.398839418077841e-05, 8.12630751170218e-05, 0.00011597294360399246, 0.00010202754492638633, 9.760419925441965e-05, 9.25893327803351e-05, 0.00011433835607022047, 0.00011002409155480564, 0.00010281284630764276, 9.539152961224318e-05, 0.000115685943455901, 0.0001112266254494898, 0.0001040231654769741, 9.805231093196198e-05, 0.00011283658386673778, 0.00010958212078548968, 0.00010556917550275102, 0.00010326960909878835, 0.00011777300824178383, 0.0001127776995417662, 0.00011168816854478791, 0.00010768866923172027, 0.00011747513053705916, 0.00011358650226611644, 0.00011256431753281504, 0.00011341700883349404, 0.00012002069706795737, 0.00011895700299646705, 0.00011628268111962825, 0.00011484880815260112 ] } } } }, "selector": { "expected_uniform_count": 1, "identities_unique": true, "identity_order_sha256": "97fd9721a263024f6a746638f32e0ad9a988cd4c111c00e9e178c1022e5c9d5e", "passed": true, "uniform_count": 1, "uniform_indices": [ 64 ], "visit_count": 65, "visits": [ { "branch": "attention", "coefficient": "detached_learned", "group": 1, "index": 0, "kind": "depth", "layer": 1, "offset": 1, "sources": 1, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 1, "index": 1, "kind": "depth", "layer": 1, "offset": 1, "sources": 2, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 1, "index": 2, "kind": "depth", "layer": 2, "offset": 2, "sources": 2, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 1, "index": 3, "kind": "depth", "layer": 2, "offset": 2, "sources": 2, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 1, "index": 4, "kind": "depth", "layer": 3, "offset": 3, "sources": 2, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 1, "index": 5, "kind": "depth", "layer": 3, "offset": 3, "sources": 2, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 1, "index": 6, "kind": "depth", "layer": 4, "offset": 4, "sources": 2, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 1, "index": 7, "kind": "depth", "layer": 4, "offset": 4, "sources": 2, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 2, "index": 8, "kind": "depth", "layer": 5, "offset": 1, "sources": 2, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 2, "index": 9, "kind": "depth", "layer": 5, "offset": 1, "sources": 3, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 2, "index": 10, "kind": "depth", "layer": 6, "offset": 2, "sources": 3, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 2, "index": 11, "kind": "depth", "layer": 6, "offset": 2, "sources": 3, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 2, "index": 12, "kind": "depth", "layer": 7, "offset": 3, "sources": 3, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 2, "index": 13, "kind": "depth", "layer": 7, "offset": 3, "sources": 3, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 2, "index": 14, "kind": "depth", "layer": 8, "offset": 4, "sources": 3, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 2, "index": 15, "kind": "depth", "layer": 8, "offset": 4, "sources": 3, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 3, "index": 16, "kind": "depth", "layer": 9, "offset": 1, "sources": 3, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 3, "index": 17, "kind": "depth", "layer": 9, "offset": 1, "sources": 4, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 3, "index": 18, "kind": "depth", "layer": 10, "offset": 2, "sources": 4, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 3, "index": 19, "kind": "depth", "layer": 10, "offset": 2, "sources": 4, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 3, "index": 20, "kind": "depth", "layer": 11, "offset": 3, "sources": 4, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 3, "index": 21, "kind": "depth", "layer": 11, "offset": 3, "sources": 4, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 3, "index": 22, "kind": "depth", "layer": 12, "offset": 4, "sources": 4, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 3, "index": 23, "kind": "depth", "layer": 12, "offset": 4, "sources": 4, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 4, "index": 24, "kind": "depth", "layer": 13, "offset": 1, "sources": 4, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 4, "index": 25, "kind": "depth", "layer": 13, "offset": 1, "sources": 5, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 4, "index": 26, "kind": "depth", "layer": 14, "offset": 2, "sources": 5, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 4, "index": 27, "kind": "depth", "layer": 14, "offset": 2, "sources": 5, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 4, "index": 28, "kind": "depth", "layer": 15, "offset": 3, "sources": 5, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 4, "index": 29, "kind": "depth", "layer": 15, "offset": 3, "sources": 5, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 4, "index": 30, "kind": "depth", "layer": 16, "offset": 4, "sources": 5, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 4, "index": 31, "kind": "depth", "layer": 16, "offset": 4, "sources": 5, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 5, "index": 32, "kind": "depth", "layer": 17, "offset": 1, "sources": 5, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 5, "index": 33, "kind": "depth", "layer": 17, "offset": 1, "sources": 6, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 5, "index": 34, "kind": "depth", "layer": 18, "offset": 2, "sources": 6, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 5, "index": 35, "kind": "depth", "layer": 18, "offset": 2, "sources": 6, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 5, "index": 36, "kind": "depth", "layer": 19, "offset": 3, "sources": 6, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 5, "index": 37, "kind": "depth", "layer": 19, "offset": 3, "sources": 6, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 5, "index": 38, "kind": "depth", "layer": 20, "offset": 4, "sources": 6, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 5, "index": 39, "kind": "depth", "layer": 20, "offset": 4, "sources": 6, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 6, "index": 40, "kind": "depth", "layer": 21, "offset": 1, "sources": 6, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 6, "index": 41, "kind": "depth", "layer": 21, "offset": 1, "sources": 7, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 6, "index": 42, "kind": "depth", "layer": 22, "offset": 2, "sources": 7, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 6, "index": 43, "kind": "depth", "layer": 22, "offset": 2, "sources": 7, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 6, "index": 44, "kind": "depth", "layer": 23, "offset": 3, "sources": 7, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 6, "index": 45, "kind": "depth", "layer": 23, "offset": 3, "sources": 7, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 6, "index": 46, "kind": "depth", "layer": 24, "offset": 4, "sources": 7, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 6, "index": 47, "kind": "depth", "layer": 24, "offset": 4, "sources": 7, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 7, "index": 48, "kind": "depth", "layer": 25, "offset": 1, "sources": 7, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 7, "index": 49, "kind": "depth", "layer": 25, "offset": 1, "sources": 8, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 7, "index": 50, "kind": "depth", "layer": 26, "offset": 2, "sources": 8, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 7, "index": 51, "kind": "depth", "layer": 26, "offset": 2, "sources": 8, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 7, "index": 52, "kind": "depth", "layer": 27, "offset": 3, "sources": 8, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 7, "index": 53, "kind": "depth", "layer": 27, "offset": 3, "sources": 8, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 7, "index": 54, "kind": "depth", "layer": 28, "offset": 4, "sources": 8, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 7, "index": 55, "kind": "depth", "layer": 28, "offset": 4, "sources": 8, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 8, "index": 56, "kind": "depth", "layer": 29, "offset": 1, "sources": 8, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 8, "index": 57, "kind": "depth", "layer": 29, "offset": 1, "sources": 9, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 8, "index": 58, "kind": "depth", "layer": 30, "offset": 2, "sources": 9, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 8, "index": 59, "kind": "depth", "layer": 30, "offset": 2, "sources": 9, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 8, "index": 60, "kind": "depth", "layer": 31, "offset": 3, "sources": 9, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 8, "index": 61, "kind": "depth", "layer": 31, "offset": 3, "sources": 9, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 8, "index": 62, "kind": "depth", "layer": 32, "offset": 4, "sources": 9, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 8, "index": 63, "kind": "depth", "layer": 32, "offset": 4, "sources": 9, "use_uniform": false }, { "branch": "output", "coefficient": "uniform", "group": null, "index": 64, "kind": "output", "layer": null, "offset": null, "sources": 9, "use_uniform": true } ], "visits_sha256": "260d93ecd556b53e98ae4b3aff8f489a0abe28f33f7d265e812410ecff18ed77" } } }, "loss_scale_gate": { "checks": { "attention_branch_output": { "batch_mean_rms": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "element_rms": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "global_l2": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "token_mean_rms": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "token_rms_mean": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "token_rms_median": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "token_rms_p95": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 } }, "mlp_branch_output": { "batch_mean_rms": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "element_rms": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "global_l2": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "token_mean_rms": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "token_rms_mean": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "token_rms_median": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "token_rms_p95": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 } }, "post_attention_state": { "batch_mean_rms": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "element_rms": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "global_l2": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "token_mean_rms": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "token_rms_mean": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "token_rms_median": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "token_rms_p95": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 } }, "post_mlp_state": { "batch_mean_rms": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "element_rms": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "global_l2": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "token_mean_rms": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "token_rms_mean": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "token_rms_median": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "token_rms_p95": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 } }, "pre_attention_input": { "batch_mean_rms": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "element_rms": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "global_l2": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "token_mean_rms": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "token_rms_mean": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "token_rms_median": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "token_rms_p95": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 } }, "pre_mlp_input": { "batch_mean_rms": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "element_rms": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "global_l2": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "token_mean_rms": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "token_rms_mean": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "token_rms_median": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 }, "token_rms_p95": { "cv_absolute_error": 0.0, "max_normalized_absolute_error": 0.0, "max_scale_ratio_error": 0.0, "passed": true, "spike_contrast_absolute_error": 0.0 } } }, "passed": true }, "parent_learned": { "bits_per_byte": 7.959341947457508, "forward": { "activation_sha256": { "attention_branch_output": "676759a2d7c0615ad7e1f246e90927fb829d00c8b6c3745032ae6b27ab7d902a", "mlp_branch_output": "afe29f3ef4d2992353f0bb59fe7c5c2c5050fca671a53184825a8d63cd83a2bc", "post_attention_state": "31853ffb07b24cc59e774bb20656840b4358b3080b42516d380f2dc0f46404c8", "post_mlp_state": "7fd9c911e93175896ad618842c120e77b3f017c17b7eec9c2bfb01fc6dbc8ac2", "pre_attention_input": "de71d4cf2d509a1063f6eac7f50c4885e9a058dee7e4b993ad40f5527f528d00", "pre_mlp_input": "2999a257265024350fd2665e2a8d452a03cd662193b4245d15290408ad29922d" }, "logits_sha256": "873a309a197bfd08fa523edcb440d2697903a34abc02dad1c7f5991509c3c151", "loss_nats": 5.516995429992676, "mixer_summary_sha256": "8450b24bdc777885a3357dc7b8080e7410acbe96943686433da76f379295133a" }, "loss_nats": 5.516995429992676, "loss_scale": 1.0, "mixers": [ { "branch": "attention", "entropy_mean": -0.0, "group": 1, "latest_source_mass": 1.0, "layer": 1, "max_source_mass": 1.0, "mixer_index": 0, "normalized_entropy": 1.0, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 1.0, "median": 1.0, "p05": 1.0, "p95": 1.0 } ], "sources": 1 }, { "branch": "mlp", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 1, "max_source_mass": 0.5, "mixer_index": 1, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 2, "max_source_mass": 0.5, "mixer_index": 2, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 2, "max_source_mass": 0.5, "mixer_index": 3, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 3, "max_source_mass": 0.5, "mixer_index": 4, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 3, "max_source_mass": 0.5, "mixer_index": 5, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 4, "max_source_mass": 0.5, "mixer_index": 6, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.6931471824645996, "group": 1, "latest_source_mass": 0.5, "layer": 4, "max_source_mass": 0.5, "mixer_index": 7, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "current_group_partial", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6931471824645996, "group": 2, "latest_source_mass": 0.5, "layer": 5, "max_source_mass": 0.5, "mixer_index": 8, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 }, { "label": "completed_group_1", "mean": 0.5, "median": 0.5, "p05": 0.5, "p95": 0.5 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 5, "max_source_mass": 0.3333333432674408, "mixer_index": 9, "normalized_entropy": 1.0000000180537645, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 6, "max_source_mass": 0.3333333432674408, "mixer_index": 10, "normalized_entropy": 1.0000000180537645, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 6, "max_source_mass": 0.3333333432674408, "mixer_index": 11, "normalized_entropy": 1.0000000180537645, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 7, "max_source_mass": 0.3333333432674408, "mixer_index": 12, "normalized_entropy": 1.0000000180537645, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 7, "max_source_mass": 0.3333333432674408, "mixer_index": 13, "normalized_entropy": 1.0000000180537645, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 8, "max_source_mass": 0.3333333432674408, "mixer_index": 14, "normalized_entropy": 1.0000000180537645, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0986123085021973, "group": 2, "latest_source_mass": 0.3333333432674408, "layer": 8, "max_source_mass": 0.3333333432674408, "mixer_index": 15, "normalized_entropy": 1.0000000180537645, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "current_group_partial", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0986123085021973, "group": 3, "latest_source_mass": 0.3333333432674408, "layer": 9, "max_source_mass": 0.3333333432674408, "mixer_index": 16, "normalized_entropy": 1.0000000180537645, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_1", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 }, { "label": "completed_group_2", "mean": 0.3333333432674408, "median": 0.3333333432674408, "p05": 0.3333333432674408, "p95": 0.3333333432674408 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 9, "max_source_mass": 0.25, "mixer_index": 17, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 10, "max_source_mass": 0.25, "mixer_index": 18, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 10, "max_source_mass": 0.25, "mixer_index": 19, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 11, "max_source_mass": 0.25, "mixer_index": 20, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 11, "max_source_mass": 0.25, "mixer_index": 21, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 12, "max_source_mass": 0.25, "mixer_index": 22, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.3862943649291992, "group": 3, "latest_source_mass": 0.25, "layer": 12, "max_source_mass": 0.25, "mixer_index": 23, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "current_group_partial", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3862943649291992, "group": 4, "latest_source_mass": 0.25, "layer": 13, "max_source_mass": 0.25, "mixer_index": 24, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_1", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_2", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 }, { "label": "completed_group_3", "mean": 0.25, "median": 0.25, "p05": 0.25, "p95": 0.25 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 13, "max_source_mass": 0.20000000298023224, "mixer_index": 25, "normalized_entropy": 1.0000000186840277, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 14, "max_source_mass": 0.20000000298023224, "mixer_index": 26, "normalized_entropy": 1.0000000186840277, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 14, "max_source_mass": 0.20000000298023224, "mixer_index": 27, "normalized_entropy": 1.0000000186840277, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 15, "max_source_mass": 0.20000000298023224, "mixer_index": 28, "normalized_entropy": 1.0000000186840277, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 15, "max_source_mass": 0.20000000298023224, "mixer_index": 29, "normalized_entropy": 1.0000000186840277, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 16, "max_source_mass": 0.20000000298023224, "mixer_index": 30, "normalized_entropy": 1.0000000186840277, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.6094379425048828, "group": 4, "latest_source_mass": 0.20000000298023224, "layer": 16, "max_source_mass": 0.20000000298023224, "mixer_index": 31, "normalized_entropy": 1.0000000186840277, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "current_group_partial", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.6094379425048828, "group": 5, "latest_source_mass": 0.20000000298023224, "layer": 17, "max_source_mass": 0.20000000298023224, "mixer_index": 32, "normalized_entropy": 1.0000000186840277, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_1", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_2", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_3", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 }, { "label": "completed_group_4", "mean": 0.20000000298023224, "median": 0.20000000298023224, "p05": 0.20000000298023224, "p95": 0.20000000298023224 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 17, "max_source_mass": 0.1666666716337204, "mixer_index": 33, "normalized_entropy": 1.0000001451965654, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 18, "max_source_mass": 0.1666666716337204, "mixer_index": 34, "normalized_entropy": 1.0000001451965654, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 18, "max_source_mass": 0.1666666716337204, "mixer_index": 35, "normalized_entropy": 1.0000001451965654, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 19, "max_source_mass": 0.1666666716337204, "mixer_index": 36, "normalized_entropy": 1.0000001451965654, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 19, "max_source_mass": 0.1666666716337204, "mixer_index": 37, "normalized_entropy": 1.0000001451965654, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 20, "max_source_mass": 0.1666666716337204, "mixer_index": 38, "normalized_entropy": 1.0000001451965654, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.791759729385376, "group": 5, "latest_source_mass": 0.1666666716337204, "layer": 20, "max_source_mass": 0.1666666716337204, "mixer_index": 39, "normalized_entropy": 1.0000001451965654, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "current_group_partial", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.791759729385376, "group": 6, "latest_source_mass": 0.1666666716337204, "layer": 21, "max_source_mass": 0.1666666716337204, "mixer_index": 40, "normalized_entropy": 1.0000001451965654, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_1", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_2", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_3", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_4", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 }, { "label": "completed_group_5", "mean": 0.1666666716337204, "median": 0.1666666716337204, "p05": 0.1666666716337204, "p95": 0.1666666716337204 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 21, "max_source_mass": 0.1428571492433548, "mixer_index": 41, "normalized_entropy": 1.00000003408302, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 22, "max_source_mass": 0.1428571492433548, "mixer_index": 42, "normalized_entropy": 1.00000003408302, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 22, "max_source_mass": 0.1428571492433548, "mixer_index": 43, "normalized_entropy": 1.00000003408302, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 23, "max_source_mass": 0.1428571492433548, "mixer_index": 44, "normalized_entropy": 1.00000003408302, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 23, "max_source_mass": 0.1428571492433548, "mixer_index": 45, "normalized_entropy": 1.00000003408302, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 24, "max_source_mass": 0.1428571492433548, "mixer_index": 46, "normalized_entropy": 1.00000003408302, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.9459102153778076, "group": 6, "latest_source_mass": 0.1428571492433548, "layer": 24, "max_source_mass": 0.1428571492433548, "mixer_index": 47, "normalized_entropy": 1.00000003408302, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "current_group_partial", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9459102153778076, "group": 7, "latest_source_mass": 0.1428571492433548, "layer": 25, "max_source_mass": 0.1428571492433548, "mixer_index": 48, "normalized_entropy": 1.00000003408302, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_1", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_2", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_3", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_4", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_5", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 }, { "label": "completed_group_6", "mean": 0.1428571492433548, "median": 0.1428571492433548, "p05": 0.1428571492433548, "p95": 0.1428571492433548 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 25, "max_source_mass": 0.125, "mixer_index": 49, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 26, "max_source_mass": 0.125, "mixer_index": 50, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 26, "max_source_mass": 0.125, "mixer_index": 51, "normalized_entropy": 1.0000000027478353, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 27, "max_source_mass": 0.125, "mixer_index": 52, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 27, "max_source_mass": 0.125, "mixer_index": 53, "normalized_entropy": 1.0000000027478353, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 28, "max_source_mass": 0.125, "mixer_index": 54, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.079441547393799, "group": 7, "latest_source_mass": 0.125, "layer": 28, "max_source_mass": 0.125, "mixer_index": 55, "normalized_entropy": 1.0000000027478353, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "current_group_partial", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.079441547393799, "group": 8, "latest_source_mass": 0.125, "layer": 29, "max_source_mass": 0.125, "mixer_index": 56, "normalized_entropy": 1.0000000027478353, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_1", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_2", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_3", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_4", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_5", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_6", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 }, { "label": "completed_group_7", "mean": 0.125, "median": 0.125, "p05": 0.125, "p95": 0.125 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 29, "max_source_mass": 0.1111111119389534, "mixer_index": 57, "normalized_entropy": 1.0000000180537645, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 30, "max_source_mass": 0.1111111119389534, "mixer_index": 58, "normalized_entropy": 1.0000000180537645, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 30, "max_source_mass": 0.1111111119389534, "mixer_index": 59, "normalized_entropy": 1.0000000180537645, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 31, "max_source_mass": 0.1111111119389534, "mixer_index": 60, "normalized_entropy": 1.0000000180537645, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 31, "max_source_mass": 0.1111111119389534, "mixer_index": 61, "normalized_entropy": 1.0000000180537645, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 32, "max_source_mass": 0.1111111119389534, "mixer_index": 62, "normalized_entropy": 1.0000000180537645, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.1972246170043945, "group": 8, "latest_source_mass": 0.1111111119389534, "layer": 32, "max_source_mass": 0.1111111119389534, "mixer_index": 63, "normalized_entropy": 1.0000000180537645, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "current_group_partial", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 }, { "branch": "output", "entropy_mean": 2.1972246170043945, "group": null, "latest_source_mass": 0.1111111119389534, "layer": null, "max_source_mass": 0.1111111119389534, "mixer_index": null, "normalized_entropy": 1.0000000180537645, "offset": null, "source_summaries": [ { "label": "embedding", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_1", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_2", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_3", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_4", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_5", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_6", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_7", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 }, { "label": "completed_group_8", "mean": 0.1111111119389534, "median": 0.1111111119389534, "p05": 0.1111111119389534, "p95": 0.1111111119389534 } ], "sources": 9 } ], "mode": "learned", "optimizer_state_after": "b82a349700eda58e2330a22ca2ee1f7fb1406efd47fd9aa306122b368a114192", "optimizer_state_before": "b82a349700eda58e2330a22ca2ee1f7fb1406efd47fd9aa306122b368a114192", "optimizer_state_unchanged": true, "positions": { "attention_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "676759a2d7c0615ad7e1f246e90927fb829d00c8b6c3745032ae6b27ab7d902a", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.0022265110164880753, 0.0021613112185150385, 0.0025601957459002733, 0.0025323182344436646, 0.0027253651060163975, 0.002904455177485943, 0.0029208383057266474, 0.00299624796025455, 0.0030937574338167906, 0.0033804215490818024, 0.0034370317589491606, 0.0038969130255281925, 0.004255065228790045, 0.004119740333408117, 0.003970506135374308, 0.004385598469525576, 0.0041999719105660915, 0.005232699681073427, 0.004664790350943804, 0.004898969549685717, 0.005131533369421959, 0.005069608334451914, 0.0051318020559847355, 0.0058372304774820805, 0.0059690922498703, 0.00554921617731452, 0.0058550527319312096, 0.006255472078919411, 0.006502739619463682, 0.006122888531535864, 0.007071305997669697, 0.006241494789719582 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.00015962477027642308, "first_to_last_ratio": 3.7699040072151497, "imbalance_abs_log_ratio": 1.3270495388586014, "last_quartile_mean": 4.234186599205714e-05, "mean": 8.617327489446325e-05, "normalized": [ 2.5428430303860874, 2.3222064178412967, 2.1011513502014654, 1.8911470095267005, 1.7004447966680725, 1.5353700031893112, 1.4229335846259188, 1.3028620492588128, 1.2049362956920768, 1.1705197150696323, 1.0773612761873523, 1.0133200843661747, 0.9906112963967714, 0.9272530070793641, 0.8550207003106389, 0.8185600560490035, 0.7703816559177782, 0.7173918665339755, 0.6760157258502337, 0.6514535263006792, 0.6184138115448249, 0.6053606047045522, 0.5838821473278404, 0.5697017460729616, 0.5457761369570343, 0.5276086337508958, 0.5104655265174319, 0.4951752820428687, 0.47527333481689216, 0.465789239220051, 0.45896414036514044, 0.45180594922816086 ], "peak_layer": 1, "peak_normalized": 2.5428430303860874, "population_cv": 0.5705341151299013, "rest_mean": 9.280180475562466e-05, "spike_contrast": 0.5428688997682254, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.0379213644191625e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00021912511147093028, 0.00020011213200632483, 0.00018106309289578348, 0.00016296633111778647, 0.00014653289690613747, 0.00013230786134954542, 0.0001226188469445333, 0.00011227188952034339, 0.0001038333066389896, 0.00010086751717608422, 9.283974941354245e-05, 8.732111018616706e-05, 8.536421955795959e-05, 7.990442827576771e-05, 7.367993384832516e-05, 7.053800072753802e-05, 6.63863102090545e-05, 6.182000652188435e-05, 5.825448897667229e-05, 5.6137883802875876e-05, 5.329074338078499e-05, 5.216590579948388e-05, 5.031503678765148e-05, 4.909306517220102e-05, 4.703131708083674e-05, 4.5465763832908124e-05, 4.3988486140733585e-05, 4.2670875700423494e-05, 4.095585973118432e-05, 4.013858415419236e-05, 3.955044303438626e-05, 3.893359826179221e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.00021442661636683624, "first_to_last_ratio": 1.843144421770586, "imbalance_abs_log_ratio": 0.6114730379455638, "last_quartile_mean": 0.0001163373926829081, "mean": 0.00015221274247778638, "normalized": [ 1.7741446897870732, 1.648461275666186, 1.5301464224684427, 1.4217121808823485, 1.3265065496562456, 1.2454420544341354, 1.1897998831173078, 1.1336248235930768, 1.0869089599614004, 1.0643003125599004, 1.0227030055526172, 0.9941765648180964, 0.9788958472869692, 0.9511102301475627, 0.9228060651723192, 0.9052418824754537, 0.8846484396889256, 0.865292475366091, 0.8497311645646503, 0.8388691935254817, 0.8256303537004397, 0.8176325372729673, 0.8076852907347736, 0.8000670639426607, 0.7892588692968658, 0.7814415503461742, 0.7731202151361662, 0.7664994567889658, 0.758490072461763, 0.7529625284402992, 0.7485174912671357, 0.7441725498875049 ], "peak_layer": 1, "peak_normalized": 1.7741446897870732, "population_cv": 0.27364872432853743, "rest_mean": 0.00015762320577373935, "spike_contrast": 0.7803181014868814, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012299624067964033, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00027004742878489196, 0.0002509168116375804, 0.0002329077833564952, 0.00021640271006617695, 0.00020191119983792305, 0.00018957215070258826, 0.0001811027032090351, 0.00017255214333999902, 0.00016544139361940324, 0.00016200006939470768, 0.00015566842921543866, 0.0001513263414381072, 0.00014900042151566595, 0.0001447710965294391, 0.00014046284195501357, 0.0001377893495373428, 0.00013465476513374597, 0.00013170854072086513, 0.00012933991092722863, 0.0001276865805266425, 0.00012567146040964872, 0.00012445409083738923, 0.00012293999316170812, 0.00012178040196886286, 0.00012013525702059269, 0.00011894536146428436, 0.00011767874821089208, 0.00011667098442558199, 0.00011545185407157987, 0.00011461049143690616, 0.00011393390013836324, 0.00011327254469506443 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.19015566632151604, "first_to_last_ratio": 1.8431444167877922, "imbalance_abs_log_ratio": 0.6114730352421437, "last_quartile_mean": 0.10316916275769472, "mean": 0.13498378219082952, "normalized": [ 1.7741447141792424, 1.6484611693507762, 1.5301464164964773, 1.421712099711509, 1.3265065258282553, 1.245442086656075, 1.1897999853143832, 1.1336247998774232, 1.0869090145314644, 1.0643003539468234, 1.0227030148964142, 0.9941765578539046, 0.978895844117059, 0.9511102293531465, 0.9228060470726628, 0.9052419251832852, 0.8846484748200404, 0.8652925213936268, 0.8497311905993837, 0.8388692019864461, 0.8256303029992552, 0.8176324960789627, 0.8076852727084025, 0.800067049482784, 0.789258877083647, 0.7814415614650243, 0.7731201949085273, 0.7664994759042474, 0.758490077799444, 0.7529625178867759, 0.7485174642151933, 0.7441725362993381 ], "peak_layer": 1, "peak_normalized": 1.7741447141792424, "population_cv": 0.2736487184871982, "rest_mean": 0.13978183573042904, "spike_contrast": 0.7803180757143806, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.10907429307699204, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.23948076367378235, 0.2225155234336853, 0.20654495060443878, 0.1919080764055252, 0.17905686795711517, 0.16811448335647583, 0.16060370206832886, 0.1530209630727768, 0.14671508967876434, 0.14366328716278076, 0.13804832100868225, 0.13419771194458008, 0.1321350634098053, 0.12838445603847504, 0.12456385046243668, 0.12219297885894775, 0.11941319704055786, 0.116800457239151, 0.11469992995262146, 0.11323373764753342, 0.11144670099020004, 0.11036712676286697, 0.10902441293001175, 0.10799607634544373, 0.10653714835643768, 0.10548193752765656, 0.10435868799686432, 0.1034649983048439, 0.10238385945558548, 0.10163772851228714, 0.10103771835565567, 0.10045122355222702 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 6.883365585963475e-05, "first_to_last_ratio": 2.2497546817252747, "imbalance_abs_log_ratio": 0.8108211799277664, "last_quartile_mean": 3.059607183786284e-05, "mean": 4.691841348858361e-05, "normalized": [ 1.7359303646705315, 1.6475859007944922, 1.5536416135247637, 1.4921074478374423, 1.436565604934822, 1.343834173349448, 1.2787439414688093, 1.2483315192759759, 1.191227587726315, 1.1603086071531024, 1.0995306735006558, 1.059977946939621, 1.0583984121521266, 1.0176934649371145, 0.9754909410255406, 0.9631621800279858, 0.9242301483750994, 0.8752327792179452, 0.8454160845678106, 0.8232909667825862, 0.7863453139125789, 0.77746685672799, 0.7497386589361914, 0.7388508716017261, 0.7093180450413769, 0.6908354487677191, 0.6685471901846501, 0.6592354484095886, 0.631440334976849, 0.6235901145414114, 0.6199506557967245, 0.6139807028410055 ], "peak_layer": 1, "peak_normalized": 1.7359303646705315, "population_cv": 0.3220412433027962, "rest_mean": 4.907019663530971e-05, "spike_contrast": 0.7193528234378936, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.529878449626267e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 8.144709863699973e-05, 7.730211655143648e-05, 7.289439963642508e-05, 7.000731420703232e-05, 6.740137905580923e-05, 6.305056740529835e-05, 5.999663699185476e-05, 5.856973439222202e-05, 5.5890508519951254e-05, 5.4439839004771784e-05, 5.1588234782684594e-05, 4.9732483603293076e-05, 4.965837433701381e-05, 4.774856279254891e-05, 4.576848732540384e-05, 4.519004141911864e-05, 4.3363412260077894e-05, 4.106453343410976e-05, 3.966558142565191e-05, 3.862750600092113e-05, 3.6894074582960457e-05, 3.647751145763323e-05, 3.517654840834439e-05, 3.4665710700210184e-05, 3.328007733216509e-05, 3.241290323785506e-05, 3.136717350571416e-05, 3.0930281354812905e-05, 2.9626178729813546e-05, 2.925785884144716e-05, 2.9087101211189292e-05, 2.8807000489905477e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 0.0001734125216898974, "first_to_last_ratio": 1.495517223041696, "imbalance_abs_log_ratio": 0.4024721155990803, "last_quartile_mean": 0.00011595488103921525, "mean": 0.00013990532806928968, "normalized": [ 1.3768889945483427, 1.3340940553978193, 1.2910049686488183, 1.2527203273458163, 1.2147536525239127, 1.1769854337196524, 1.14744939519443, 1.1220955988146075, 1.0944022472959734, 1.0773022658134643, 1.0508559212107247, 1.03206209473486, 1.0208171898483585, 1.000561882500425, 0.9805161605170362, 0.9663129296065233, 0.9487424991891081, 0.931931984738623, 0.9181974345490227, 0.9075104531838204, 0.8943640941785878, 0.8862664021756573, 0.8757830774574916, 0.86790401991425, 0.8562894022841879, 0.8478155487572495, 0.8387227683262297, 0.8314011142413483, 0.8224071457687988, 0.8161875050523302, 0.8112199678685613, 0.8064334645939688 ], "peak_layer": 1, "peak_normalized": 1.3768889945483427, "population_cv": 0.16325957486062354, "rest_mean": 0.0001431148236776978, "spike_contrast": 0.8564734849544942, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001225740517838858, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0001926341064972803, 0.00018664686649572104, 0.00018061847367789596, 0.00017526224837638438, 0.0001699505082797259, 0.00016466653323732316, 0.00016053428407758474, 0.00015698715287726372, 0.00015311270544771105, 0.00015072032692842185, 0.00014702034241054207, 0.00014439098595175892, 0.00014281776384450495, 0.00013998393842484802, 0.00013717943511437625, 0.00013519232743419707, 0.00013273413060232997, 0.00013038225006312132, 0.00012846071331296116, 0.00012696554767899215, 0.0001251263020094484, 0.00012399339175317436, 0.0001225267187692225, 0.00012142439663875848, 0.00011979944974882528, 0.00011861391249112785, 0.00011734178406186402, 0.00011631744564510882, 0.00011505914153531194, 0.00011418898066040128, 0.00011349399574100971, 0.00011282433843007311 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 0.00014675117381557357, "first_to_last_ratio": 1.2676837465168667, "imbalance_abs_log_ratio": 0.23719141364257146, "last_quartile_mean": 0.00011576323686313117, "mean": 0.00013042020850662084, "normalized": [ 1.169988069421413, 1.156175152925158, 1.1417665260225645, 1.1306726331060184, 1.1183621029407895, 1.1062694829504214, 1.0940115056230622, 1.0844989956600726, 1.0711591662817939, 1.0579765491085218, 1.0443900226505625, 1.035471661013805, 1.0260498632741444, 1.013992390086173, 1.0062479316494695, 0.9977001177294099, 0.9863163473759529, 0.9769959732660659, 0.9687585669786032, 0.9602628595878113, 0.9503845997255937, 0.9426104617668664, 0.9330243108845692, 0.9259760938667744, 0.914797625470175, 0.906289421438531, 0.8956357003805738, 0.888470885240542, 0.8806216558595509, 0.8756991509238615, 0.8712858841028995, 0.8681382926882502 ], "peak_layer": 1, "peak_normalized": 1.169988069421413, "population_cv": 0.09123006364857297, "rest_mean": 0.00013202972385678786, "spike_contrast": 0.9219804606102014, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012172882561571896, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00015259008796419948, 0.00015078860451467335, 0.00014890942838974297, 0.0001474625605624169, 0.00014585701865144074, 0.00014427989663090557, 0.00014268120867200196, 0.00014144058513920754, 0.00013970080181024969, 0.0001379815221298486, 0.00013620956451632082, 0.00013504642993211746, 0.0001338176371064037, 0.00013224509893916547, 0.00013123506505507976, 0.0001301202573813498, 0.00012863558367826045, 0.00012742001854348928, 0.00012634569429792464, 0.00012523768236860633, 0.00012394935765769333, 0.00012293545296415687, 0.00012168522516731173, 0.00012076599523425102, 0.00011930809705518186, 0.00011819845531135798, 0.00011680899478960782, 0.00011587455810513347, 0.00011485085997264832, 0.00011420886585256085, 0.00011363328667357564, 0.00011322277714498341 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00030002388848515693, "first_to_last_ratio": 2.2836575149168885, "imbalance_abs_log_ratio": 0.8257783308640597, "last_quartile_mean": 0.0001313786706305109, "mean": 0.0001947244400071213, "normalized": [ 1.8998833386658438, 1.7835645331795598, 1.6713073472446418, 1.5703972875779748, 1.4780250735507643, 1.3761669788219328, 1.3066447563403325, 1.2401013786965793, 1.1762106459027857, 1.150547637187856, 1.0768735950666006, 1.0264900078881174, 1.0140784159549234, 0.9666871251601796, 0.9096841069789114, 0.8754807117769676, 0.8421876125778792, 0.8073809902734768, 0.77911667677541, 0.7607651063686129, 0.7403822030704592, 0.7308855611321243, 0.7152791523187013, 0.7043381880055872, 0.6944157299089508, 0.686864629891504, 0.680599049276497, 0.6764772714226681, 0.6702412094778135, 0.6672258960044359, 0.6635863560708457, 0.6581114274310631 ], "peak_layer": 1, "peak_normalized": 1.8998833386658438, "population_cv": 0.3592853832280775, "rest_mean": 0.00020492727344390005, "spike_contrast": 0.6813594750078023, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013962913944851608, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0003699537192005664, 0.0003473036049399525, 0.0003254443872720003, 0.00030579473241232336, 0.00028780760476365685, 0.00026797334430739284, 0.0002544356684666127, 0.0002414780465187505, 0.0002290369593538344, 0.00022403974435292184, 0.00020969360775779933, 0.00019988269195891917, 0.0001974658516701311, 0.00018823760910890996, 0.00017713772831484675, 0.00017047749133780599, 0.00016399451124016196, 0.0001572168112033978, 0.00015171305858530104, 0.00014813955931458622, 0.00014417050988413393, 0.00014232128160074353, 0.00013928233238402754, 0.0001371518592350185, 0.00013521971413865685, 0.00013374933041632175, 0.00013252926873974502, 0.00013172665785532445, 0.00013051234418526292, 0.00012992518895771354, 0.00012921648158226162, 0.000128150379168801 ] } } }, "mlp_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "afe29f3ef4d2992353f0bb59fe7c5c2c5050fca671a53184825a8d63cd83a2bc", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.0027073738165199757, 0.002722575329244137, 0.0026850115973502398, 0.0026861224323511124, 0.00271838391199708, 0.0026645169127732515, 0.0026859452482312918, 0.0027359756641089916, 0.0026767742820084095, 0.0027451682835817337, 0.0026999011170119047, 0.002663193503394723, 0.002626648638397455, 0.0026531771291047335, 0.0026517712976783514, 0.002676903735846281, 0.002670086920261383, 0.0026355870068073273, 0.0026610493659973145, 0.0026388936676084995, 0.0026469926815479994, 0.00264862016774714, 0.002641494618728757, 0.002674198243767023, 0.002557571977376938, 0.00252733426168561, 0.0025468147359788418, 0.0026856842450797558, 0.002551089273765683, 0.0025778315030038357, 0.0026001501828432083, 0.0025662274565547705 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.00015858857113926206, "first_to_last_ratio": 3.757867690304466, "imbalance_abs_log_ratio": 1.3238516928952826, "last_quartile_mean": 4.220174423608114e-05, "mean": 8.573221236929385e-05, "normalized": [ 2.5461641896458365, 2.3032502265274117, 2.0924715218207024, 1.8886722264368014, 1.7031663195050915, 1.537434754996766, 1.4223955235407997, 1.3049501874408618, 1.2055297161045848, 1.1692542365736316, 1.0769571962730058, 1.0144505847411098, 0.9913601784485884, 0.9317001386730132, 0.857757082878185, 0.8155634736467603, 0.7682612945963089, 0.7217806418748633, 0.678338533281248, 0.6511820384364709, 0.6192882306865346, 0.6053793146733001, 0.5848721334223365, 0.5718144052517429, 0.5462281883927541, 0.527515083157495, 0.5112993585545178, 0.496920907994256, 0.47722358366421574, 0.46552107689700334, 0.4601234027970099, 0.45317424906679377 ], "peak_layer": 1, "peak_normalized": 2.5461641896458365, "population_cv": 0.5686091737198813, "rest_mean": 9.231269224848668e-05, "spike_contrast": 0.5437781067692293, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.0197621021652594e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00021828828903380781, 0.00019746273756027222, 0.00017939221288543195, 0.0001619200484128669, 0.00014601621660403907, 0.000131807682919316, 0.00012194511509733275, 0.00011187626660102978, 0.00010335272963857278, 0.00010024275252362713, 9.232992306351662e-05, 8.697109296917915e-05, 8.499150135321543e-05, 7.987671415321529e-05, 7.353741239057854e-05, 6.992006092332304e-05, 6.586474046343938e-05, 6.187985127326101e-05, 5.8155463193543255e-05, 5.582727681030519e-05, 5.309295011102222e-05, 5.1900507969548926e-05, 5.0142381951445714e-05, 4.902291402686387e-05, 4.682935104938224e-05, 4.522503513726406e-05, 4.3834825191879645e-05, 4.260212881490588e-05, 4.0913433622336015e-05, 3.991015182691626e-05, 3.944739728467539e-05, 3.885163096128963e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.00021242323236947414, "first_to_last_ratio": 1.8345651540958352, "imbalance_abs_log_ratio": 0.6068074801791611, "last_quartile_mean": 0.0001157894184871111, "mean": 0.00015111673860701558, "normalized": [ 1.771707915810251, 1.6366668487510905, 1.5242909365839221, 1.4186705636795862, 1.3264449439106374, 1.2453857912414694, 1.1893415085703798, 1.1330085309323608, 1.0858463851390703, 1.062861239127963, 1.023237806768965, 0.9942126959777651, 0.978436257513136, 0.9531027445378829, 0.9232414968561398, 0.904013237249231, 0.8853704564278231, 0.8680164030054869, 0.8515402788738432, 0.8395817783841374, 0.8267435189825344, 0.8185143642831039, 0.809172703595369, 0.8007918347797284, 0.7909896391756025, 0.7821052393452608, 0.7753388667989797, 0.7682453290925442, 0.7604995795745633, 0.7549901567959042, 0.7508572350862155, 0.7467737131490535 ], "peak_layer": 1, "peak_normalized": 1.771707915810251, "population_cv": 0.2714170004285523, "rest_mean": 0.00015645500981783562, "spike_contrast": 0.7816309251520459, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012229007406858728, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0002677347220014781, 0.00024732775636948645, 0.00023034587502479553, 0.00021438486874103546, 0.00020044803386554122, 0.0001881986390799284, 0.00017972940986510366, 0.00017121655400842428, 0.00016408956435043365, 0.00016061612404882908, 0.0001546283601783216, 0.00015024218009784818, 0.00014785809617023915, 0.0001440297783119604, 0.00013951724395155907, 0.000136611532070674, 0.0001337942958343774, 0.00013117180787958205, 0.0001286819897359237, 0.00012687486014328897, 0.00012493478425312787, 0.00012369122123345733, 0.00012227953993715346, 0.00012101305037504062, 0.00011953177454415709, 0.00011818919301731512, 0.00011716668086592108, 0.00011609472858253866, 0.00011492421617731452, 0.00011409165017539635, 0.00011346709652571008, 0.00011285000800853595 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.18837904930114746, "first_to_last_ratio": 1.8345651581276443, "imbalance_abs_log_ratio": 0.6068074823768531, "last_quartile_mean": 0.10268321540206671, "mean": 0.13401183625683188, "normalized": [ 1.7717078885231814, 1.6366668119471057, 1.524290957183596, 1.418670514273678, 1.326444920903592, 1.2453858718827995, 1.1893415500975306, 1.1330085716119305, 1.0858463365841577, 1.062861212873729, 1.0232377434953237, 0.9942127340740732, 0.9784362443707608, 0.9531027271661117, 0.9232414917376033, 0.904013241653558, 0.885370477646398, 0.8680164390463408, 0.8515402681847718, 0.8395816969955481, 0.8267435345782925, 0.8185144279720658, 0.8091727254708025, 0.8007918405918669, 0.7909896328038464, 0.7821052665924377, 0.7753388466443417, 0.7682453528580236, 0.7604996016741158, 0.7549901615676585, 0.7508572336421532, 0.7467736753526057 ], "peak_layer": 1, "peak_normalized": 1.7717078885231814, "population_cv": 0.27141699472708714, "rest_mean": 0.1387458680956452, "spike_contrast": 0.7816309473985974, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.10844806432724, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.23742982745170593, 0.2193327248096466, 0.2042730301618576, 0.19011864066123962, 0.1777593195438385, 0.16689644753932953, 0.15938584506511688, 0.151836559176445, 0.1455162614583969, 0.1424359828233719, 0.13712596893310547, 0.13323627412319183, 0.13112203776836395, 0.12772704660892487, 0.1237252876162529, 0.1211484745144844, 0.11865012347698212, 0.11632447689771652, 0.11411647498607635, 0.11251388490200043, 0.11079341918230057, 0.10969062149524689, 0.10843872278928757, 0.10731558501720428, 0.10600197315216064, 0.10481136292219162, 0.10390458256006241, 0.1029539704322815, 0.10191594809293747, 0.10117761790752411, 0.10062375664710999, 0.10007651150226593 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 6.873187476230669e-05, "first_to_last_ratio": 2.248757500145603, "imbalance_abs_log_ratio": 0.8103778415280068, "last_quartile_mean": 3.0564378221242805e-05, "mean": 4.685761763312257e-05, "normalized": [ 1.7400006587426948, 1.6405194725375354, 1.5551306973978087, 1.4944078341288562, 1.4327724019957897, 1.3414187773842097, 1.2804088050749736, 1.2499327782691996, 1.1927955171655855, 1.1608931065609767, 1.0981756972550076, 1.0577423904261343, 1.0542786038595535, 1.0218640087393278, 0.9775807389332813, 0.9570972402588515, 0.9209982861589985, 0.8815092249082567, 0.8478232134430647, 0.8204952153707635, 0.7871780668180614, 0.7759097745574741, 0.7506963524238334, 0.7421155343193898, 0.7072927325438048, 0.6891814936654451, 0.6695399899996352, 0.6616343983211914, 0.6345111343049281, 0.6212687927046295, 0.6200231913288072, 0.6148038704019305 ], "peak_layer": 1, "peak_normalized": 1.7400006587426948, "population_cv": 0.3217536161653951, "rest_mean": 4.900405669845727e-05, "spike_contrast": 0.719671983430414, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.526684668031521e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 8.153228554874659e-05, 7.687083416385576e-05, 7.286971958819777e-05, 7.002439087955281e-05, 6.71363013680093e-05, 6.285568815656006e-05, 5.999690620228648e-05, 5.8568872191244736e-05, 5.5891556257847697e-05, 5.439668530016206e-05, 5.145789691596292e-05, 4.9563288484932855e-05, 4.9400983698433265e-05, 4.7882112994557247e-05, 4.580710447044112e-05, 4.484729652176611e-05, 4.3155785533599555e-05, 4.130542220082134e-05, 3.972697595600039e-05, 3.844645107164979e-05, 3.688528886414133e-05, 3.635728353401646e-05, 3.517584264045581e-05, 3.477376594673842e-05, 3.314205241622403e-05, 3.229340290999971e-05, 3.1373048841487616e-05, 3.10026116494555e-05, 2.9731680115219206e-05, 2.9111175535945222e-05, 2.9052809622953646e-05, 2.8808244678657502e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 0.0001719075116852764, "first_to_last_ratio": 1.4894635014227988, "imbalance_abs_log_ratio": 0.39841598895878805, "last_quartile_mean": 0.00011541572621354135, "mean": 0.00013894635094402474, "normalized": [ 1.3738985779819257, 1.3296082759737424, 1.2883384004275333, 1.2499940655466777, 1.2130730176151365, 1.1755503979222366, 1.147409009170091, 1.11990595257071, 1.0929037170303153, 1.075598370036652, 1.0512762916941902, 1.0312594730951, 1.0194418974669357, 1.0018996501730997, 0.9803460168062212, 0.9648004557293595, 0.9493010807852901, 0.9344788929945884, 0.9198953859218769, 0.908001147597105, 0.8952913731815082, 0.886953676542665, 0.8771894463102521, 0.8683888934041172, 0.8579399877904575, 0.8482799643655919, 0.8408636339472488, 0.8330870307470335, 0.82434015220157, 0.8181187963300758, 0.8135579418994486, 0.809009026741244 ], "peak_layer": 1, "peak_normalized": 1.3738985779819257, "population_cv": 0.16161423472023242, "rest_mean": 0.0001421073115866252, "spike_contrast": 0.8576417505420824, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012187716347398237, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0001908981939777732, 0.0001847442181315273, 0.00017900991952046752, 0.00017368211410939693, 0.00016855206922627985, 0.00016333843814209104, 0.00015942829486448318, 0.00015560684551019222, 0.0001518549834145233, 0.00014945046859793365, 0.00014607100456487387, 0.0001432897406630218, 0.00014164773165248334, 0.00013921030040364712, 0.000136215501697734, 0.0001340555027127266, 0.0001319019211223349, 0.00012984243221580982, 0.0001278161071240902, 0.00012616344611160457, 0.00012439746933523566, 0.00012323897681199014, 0.00012188227265141904, 0.00012065946793882176, 0.00011920763063244522, 0.00011786540562752634, 0.00011683493357850239, 0.00011575440294109285, 0.0001145390560850501, 0.00011367462138878182, 0.00011304090730845928, 0.00011240885214647278 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 0.00014552221000485588, "first_to_last_ratio": 1.262697055715921, "imbalance_abs_log_ratio": 0.2332499537229619, "last_quartile_mean": 0.00011524712863320019, "mean": 0.00012953460168318998, "normalized": [ 1.1676213631634658, 1.1525055688027706, 1.1391580058081985, 1.1289307982409096, 1.1166731577821778, 1.1054001772932092, 1.094813144753513, 1.0822854389705017, 1.0679308603491235, 1.0564894822224282, 1.0456838360716472, 1.0337072702557388, 1.0234945545465333, 1.0158819477934218, 1.0051772952892828, 0.9951467950359763, 0.9866643397779161, 0.9787559418439278, 0.970448736962688, 0.9607908684400951, 0.9509939230158972, 0.9428061355388626, 0.934924980051271, 0.9261034265480621, 0.9170582603044295, 0.9058562786860701, 0.8981778407016658, 0.8907564927732574, 0.8831168120837269, 0.8778216107141017, 0.8745192645108896, 0.8703053916682415 ], "peak_layer": 1, "peak_normalized": 1.1676213631634658, "population_cv": 0.08967236486753827, "rest_mean": 0.00013110875065817877, "spike_contrast": 0.9231588022206524, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012103419721825049, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00015124736819416285, 0.00014928934979252517, 0.000147560378536582, 0.00014623560127802193, 0.00014464781270362437, 0.00014318757166620344, 0.00014181618462316692, 0.00014019341324456036, 0.0001383339986205101, 0.00013685194426216185, 0.00013545223919209093, 0.00013390085950959474, 0.00013257795944809914, 0.00013159186346456409, 0.00013020524056628346, 0.00012890594371128827, 0.00012780717224813998, 0.00012678276107180864, 0.0001257066905964166, 0.0001244556624442339, 0.00012318661902099848, 0.0001221260172314942, 0.00012110513489460573, 0.0001199624384753406, 0.00011879077646881342, 0.00011733973224181682, 0.00011634510883595794, 0.00011538378748809919, 0.0001143941844929941, 0.00011370827269274741, 0.00011328050459269434, 0.0001127346622524783 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00029741139769612346, "first_to_last_ratio": 2.278072415841521, "imbalance_abs_log_ratio": 0.8233296537390045, "last_quartile_mean": 0.00013055396993877366, "mean": 0.00019328556300024502, "normalized": [ 1.889398576482247, 1.780233894901915, 1.6772594716902607, 1.566224958959574, 1.4762232610465842, 1.375571144634086, 1.3083686481730985, 1.2364401527267181, 1.173148897934763, 1.1508864184295666, 1.080457040307468, 1.0318582947152708, 1.0081957004611959, 0.9698994715426978, 0.9066127339785177, 0.8754550048258108, 0.8446146862114313, 0.8115724449181009, 0.7806677558079268, 0.7613480996239201, 0.7403134026279838, 0.7294052762839086, 0.7160186976339193, 0.7062575707090604, 0.6942777320753531, 0.6866005528973866, 0.6829938476675228, 0.6753403086482458, 0.6715967314167978, 0.6686860556999406, 0.6640286432895989, 0.6600445236791291 ], "peak_layer": 1, "peak_normalized": 1.889398576482247, "population_cv": 0.3579154260008552, "rest_mean": 0.00020340604748559632, "spike_contrast": 0.681567477924495, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013863494677934796, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0003651934675872326, 0.0003440935106482357, 0.00032419004128314555, 0.0003027286729775369, 0.0002853326441254467, 0.00026587804313749075, 0.00025288877077400684, 0.00023898603103589267, 0.00022675274522043765, 0.00022244972933549434, 0.0002088367473334074, 0.00019944331143051386, 0.00019486967357806861, 0.00018746756541077048, 0.00017523515271022916, 0.00016921281348913908, 0.0001632518251426518, 0.00015686523693148047, 0.00015089180669747293, 0.00014715759607497603, 0.00014309189282357693, 0.00014098350948188454, 0.00013839607709087431, 0.0001365093921776861, 0.0001341938623227179, 0.00013270997442305088, 0.00013201285037212074, 0.00013053353177383542, 0.0001298099523410201, 0.00012924736074637622, 0.00012834715016651899, 0.00012757707736454904 ] } } }, "post_attention_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "31853ffb07b24cc59e774bb20656840b4358b3080b42516d380f2dc0f46404c8", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.0022265110164880753, 0.0040087271481752396, 0.005379301495850086, 0.006410556845366955, 0.0027253651060163975, 0.004838948138058186, 0.006175771821290255, 0.00742313964292407, 0.0030937574338167906, 0.005150026176124811, 0.007019025273621082, 0.008143599145114422, 0.004255065228790045, 0.0066224560141563416, 0.008129785768687725, 0.010099578648805618, 0.0041999719105660915, 0.00719039561226964, 0.009030314162373543, 0.010736005380749702, 0.005131533369421959, 0.00787449348717928, 0.009912566281855106, 0.012113703414797783, 0.0059690922498703, 0.00876442901790142, 0.010806876234710217, 0.012353328987956047, 0.006502739619463682, 0.009470963850617409, 0.012123330496251583, 0.01384514570236206 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.00015962236011546338, "first_to_last_ratio": 3.7698422272252903, "imbalance_abs_log_ratio": 1.3270331510418396, "last_quartile_mean": 4.2341920561739244e-05, "mean": 8.61727822893954e-05, "normalized": [ 2.542734798705971, 2.3221448837042376, 2.101145461274622, 1.8911858524935854, 1.7004204056817789, 1.5354142426015858, 1.4229001770142031, 1.3028733810328659, 1.2049412416934622, 1.1705258152816296, 1.0773908232649334, 1.0133220774376726, 0.990609360100461, 0.9272534949358001, 0.8550385909364457, 0.8185651575045715, 0.7703972051466339, 0.7173947009730917, 0.6760256695667757, 0.6514523109028573, 0.618419077603279, 0.6053620388084041, 0.5838930841859182, 0.5697043695032278, 0.5457802700901755, 0.5276149005446632, 0.5104701332743111, 0.49517988582787786, 0.47527562953524777, 0.46579059316028215, 0.45896435764104826, 0.45181000957238127 ], "peak_layer": 1, "peak_normalized": 2.542734798705971, "population_cv": 0.5705202801130907, "rest_mean": 9.280119640472414e-05, "spike_contrast": 0.542873885449774, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.03793460666202e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00021911453222855926, 0.00020010568550787866, 0.00018106155039276928, 0.00016296874673571438, 0.00014652995741926134, 0.00013231091725174338, 0.0001226152671733871, 0.00011227222421439365, 0.00010383313929196447, 0.00010086746624438092, 9.284176485380158e-05, 8.732078276807442e-05, 8.536356472177431e-05, 7.99040135461837e-05, 7.368105434579775e-05, 7.053803710732609e-05, 6.638727063545957e-05, 6.181989738252014e-05, 5.825501284562051e-05, 5.613745815935545e-05, 5.329089253791608e-05, 5.216573117650114e-05, 5.0315691623836756e-05, 4.9093010602518916e-05, 4.703140439232811e-05, 4.546604395727627e-05, 4.398863165988587e-05, 4.267102849553339e-05, 4.095582335139625e-05, 4.013847137684934e-05, 3.955023566959426e-05, 3.893372559105046e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.00021442477373057045, "first_to_last_ratio": 1.8431254562763404, "imbalance_abs_log_ratio": 0.6114627481432037, "last_quartile_mean": 0.00011633759004325839, "mean": 0.00015221248349917005, "normalized": [ 1.7740805953138137, 1.6484457246979003, 1.5301434809496548, 1.4217241600875643, 1.3264958046529274, 1.2454577490392524, 1.1897861330408546, 1.1336265611690741, 1.086914920172498, 1.064301358569379, 1.0227209980571423, 0.9941753882589759, 0.9788979908211966, 0.9511107967643203, 0.9228143274453928, 0.9052447611165039, 0.8846530997418666, 0.8652930871753776, 0.8497381552743479, 0.8388656494639406, 0.8256334792987466, 0.8176309647342147, 0.8076951735882674, 0.8000661307359944, 0.7892605467734257, 0.78144613040264, 0.7731219607566042, 0.7664988488799842, 0.7584935618388989, 0.7529639051546407, 0.7485195296370439, 0.7441790263875565 ], "peak_layer": 1, "peak_normalized": 1.7740805953138137, "population_cv": 0.2736411578809574, "rest_mean": 0.00015762286892384981, "spike_contrast": 0.7803207938393821, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012299640220589936, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00027003721334040165, 0.00025091401766985655, 0.00023290693934541196, 0.0002164041652576998, 0.000201909220777452, 0.0001895742170745507, 0.00018110030214302242, 0.00017255211423616856, 0.00016544201935175806, 0.00016199995297938585, 0.00015567090304102749, 0.00015132590488065034, 0.0001490004942752421, 0.00014477093645837158, 0.00014046386058907956, 0.00013778955326415598, 0.0001346552453469485, 0.00013170840975362808, 0.00012934075493831187, 0.00012768582382705063, 0.00012567172234412283, 0.00012445363972801715, 0.00012294128828216344, 0.00012178005272289738, 0.00012013530795229599, 0.00011894585622940212, 0.00011767881369451061, 0.00011667069338727742, 0.00011545218876563013, 0.00011461050598882139, 0.00011393401655368507, 0.00011327333777444437 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.190154030919075, "first_to_last_ratio": 1.84312545379361, "imbalance_abs_log_ratio": 0.6114627467961817, "last_quartile_mean": 0.10316933691501617, "mean": 0.13498355145566165, "normalized": [ 1.774080628234443, 1.6484456620184311, 1.5301434020504485, 1.4217242444591272, 1.326495766996536, 1.2454576834375484, 1.189786122599316, 1.133626627255238, 1.0869149569673349, 1.0643014004698326, 1.0227209907477024, 0.9941753870542451, 0.9788979589698585, 0.9511107512164827, 0.9228143032197887, 0.9052447972745712, 0.8846531331847195, 0.8652931173486506, 0.8497382179112226, 0.8388656682549475, 0.8256334805754935, 0.8176309131099365, 0.8076951535479161, 0.8000661540388709, 0.7892605573866218, 0.7814461538045975, 0.7731219580171443, 0.7664988542586116, 0.7584935821787124, 0.7529638601652359, 0.7485195164462104, 0.7441789968002049 ], "peak_layer": 1, "peak_normalized": 1.774080628234443, "population_cv": 0.27364115345546325, "rest_mean": 0.13978153549962574, "spike_contrast": 0.7803207857775153, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.10907443761825561, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.2394717037677765, 0.22251304984092712, 0.2065441906452179, 0.19190938770771027, 0.1790551096200943, 0.16811630129814148, 0.16060155630111694, 0.1530209481716156, 0.14671564102172852, 0.1436631828546524, 0.13805051147937775, 0.13419732451438904, 0.13213512301445007, 0.1283843070268631, 0.1245647519826889, 0.12219315767288208, 0.11941362172365189, 0.11680033802986145, 0.11470068246126175, 0.1132330670952797, 0.11144693940877914, 0.11036672443151474, 0.10902556031942368, 0.10799577087163925, 0.10653719305992126, 0.10548237711191177, 0.1043587476015091, 0.10346473753452301, 0.10238415747880936, 0.10163773596286774, 0.10103782266378403, 0.10045192390680313 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 6.883367768750759e-05, "first_to_last_ratio": 2.249759357551583, "imbalance_abs_log_ratio": 0.810823258297238, "last_quartile_mean": 3.059601795030176e-05, "mean": 4.691846504556452e-05, "normalized": [ 1.7359059710084281, 1.6475789727938472, 1.5536456441204052, 1.4921194549590373, 1.4365657321885223, 1.3438513058553838, 1.2787343947807137, 1.2483299149152907, 1.1912354282513808, 1.1603083401306369, 1.099547144002948, 1.059972440022979, 1.0583955432739494, 1.0176800955770247, 0.9754963047746087, 0.9631595708780456, 0.9242400656751482, 0.8752301116137658, 0.8454245377074081, 0.8232853322610043, 0.7863494898171388, 0.7774642190168813, 0.7497486129019063, 0.7388483538626245, 0.709315327139865, 0.6908343794807076, 0.6685469983108738, 0.6592365073812129, 0.6314341746588822, 0.6235898169926835, 0.6199474545603838, 0.6139783610863121 ], "peak_layer": 1, "peak_normalized": 1.7359059710084281, "population_cv": 0.3220406003070692, "rest_mean": 4.9070239684725595e-05, "spike_contrast": 0.7193541792517962, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.52988819940947e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 8.144604362314567e-05, 7.730187644483522e-05, 7.28946688468568e-05, 7.000795449130237e-05, 6.740145909134299e-05, 6.305144052021205e-05, 5.999625500408001e-05, 5.8569723478285596e-05, 5.589093780145049e-05, 5.4439886298496276e-05, 5.158906424185261e-05, 4.973227987647988e-05, 4.9658294301480055e-05, 4.774798799189739e-05, 4.576878927764483e-05, 4.51899686595425e-05, 4.3363925215089694e-05, 4.1064453398576006e-05, 3.9666021621087566e-05, 3.86272840842139e-05, 3.689431105158292e-05, 3.6477427784120664e-05, 3.5177054087398574e-05, 3.466563066467643e-05, 3.3279986382694915e-05, 3.2412888685939834e-05, 3.136719897156581e-05, 3.093036502832547e-05, 2.9625922252307646e-05, 2.9257877031341195e-05, 2.9086982976878062e-05, 2.8806922273361124e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 0.00017341253987979144, "first_to_last_ratio": 1.4955148931299893, "imbalance_abs_log_ratio": 0.4024705576674927, "last_quartile_mean": 0.00011595507385209203, "mean": 0.00013990553861731314, "normalized": [ 1.3768818258178377, 1.3340879911950096, 1.2910054180628048, 1.2527236427111486, 1.2147542166907819, 1.1769889670731195, 1.1474411155803916, 1.1220953663134672, 1.0944075691302455, 1.0773007485615413, 1.0508678613623372, 1.0320596054419082, 1.0208161736522385, 1.000559648637946, 0.9805197815181176, 0.9663128275349431, 0.948744191771126, 0.9319305822463463, 0.9182019814348666, 0.9075038868226692, 0.8943650364968407, 0.886262676120959, 0.8757907045344768, 0.8679002174812178, 0.856288373659771, 0.8478178092769747, 0.8387215581135022, 0.8313975747674917, 0.8224084044020752, 0.8161864847722404, 0.8112200992010112, 0.8064376596145924 ], "peak_layer": 1, "peak_normalized": 1.3768818258178377, "population_cv": 0.16325888721442564, "rest_mean": 0.0001431150389921472, "spike_contrast": 0.8564734877369171, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001225742365932092, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0001926333934534341, 0.00018664629897102714, 0.00018061880837194622, 0.0001752629759721458, 0.00016995084297377616, 0.0001646672753849998, 0.00016053336730692536, 0.00015698735660407692, 0.00015311368042603135, 0.00015072034148033708, 0.00014702223415952176, 0.00014439085498452187, 0.0001428178366040811, 0.00013998383656144142, 0.00013718014815822244, 0.00013519251660909504, 0.00013273456715978682, 0.00013038225006312132, 0.00012846154277212918, 0.00012696482008323073, 0.00012512662215158343, 0.0001239930570591241, 0.00012252797023393214, 0.000121424047392793, 0.00011979948612861335, 0.00011861440725624561, 0.00011734179133782163, 0.0001163171255029738, 0.00011505949078127742, 0.00011418900976423174, 0.00011349418491590768, 0.00011282509512966499 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 0.00014675183047074825, "first_to_last_ratio": 1.2676545313577883, "imbalance_abs_log_ratio": 0.23716836728277432, "last_quartile_mean": 0.00011576642282307148, "mean": 0.00013041823922321782, "normalized": [ 1.1700698937843759, 1.1560440994564782, 1.1417398043557874, 1.130463200949706, 1.118403537277814, 1.1065304334184665, 1.0939881912841023, 1.0846815122144253, 1.0709468270677047, 1.0579907390333845, 1.0442838370166418, 1.035550673160169, 1.025959021745254, 1.0140203095307772, 1.0062047700154308, 0.9976631870190885, 0.9861765922450596, 0.9770016877798172, 0.9687702939454803, 0.9602422119882037, 0.9504995944091904, 0.9425883202531964, 0.9330014667281973, 0.9259385264566372, 0.9147443798098084, 0.906202796813466, 0.8958065504175365, 0.8885655303249116, 0.8806194435723184, 0.8757905905296257, 0.8713783728577705, 0.8681336045391749 ], "peak_layer": 1, "peak_normalized": 1.1700698937843759, "population_cv": 0.09121429133712369, "rest_mean": 0.0001320278307604086, "spike_contrast": 0.921975648780332, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001217264449223876, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0001525984553154558, 0.0001507692359155044, 0.000148903694935143, 0.00014743302017450333, 0.00014586022007279098, 0.00014431175077334046, 0.00014267601363826543, 0.00014146225294098258, 0.000139670999487862, 0.00013798128929920495, 0.00013619365927297622, 0.00013505469541996717, 0.00013380376913119107, 0.00013224674330558628, 0.00013122745440341532, 0.00013011347618885338, 0.00012861541472375393, 0.00012741883983835578, 0.0001263453159481287, 0.0001252330985153094, 0.0001239624834852293, 0.00012293070903979242, 0.00012168040848337114, 0.00012075927224941552, 0.00011929935135412961, 0.00011818537313956767, 0.00011682951299007982, 0.00011588515189941972, 0.00011484883725643158, 0.00011421906674513593, 0.00011364363308530301, 0.00011322045611450449 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00030004180916876066, "first_to_last_ratio": 2.2837506327398005, "imbalance_abs_log_ratio": 0.8258191057718037, "last_quartile_mean": 0.00013138116082700435, "mean": 0.00019473525253488333, "normalized": [ 1.9000096511818072, 1.783700741635724, 1.6708066910504589, 1.570366884717585, 1.4783058799917985, 1.376185321596543, 1.3064973297630273, 1.2402700043336092, 1.1767953849206394, 1.1505341943958838, 1.0771542571420398, 1.026098162637436, 1.013898063759585, 0.9669473025647412, 0.9095398902231254, 0.8755205777387188, 0.842301214161826, 0.8073124727024382, 0.7791991818813049, 0.760574607765185, 0.7402055397447304, 0.7308143413325056, 0.7152667122155245, 0.7043714155678553, 0.6942467749765324, 0.6868749151283189, 0.680655564574882, 0.676363713518914, 0.6701061029149729, 0.6672276319278244, 0.6636177363814285, 0.6582317375530352 ], "peak_layer": 1, "peak_normalized": 1.9000096511818072, "population_cv": 0.35930746026536914, "rest_mean": 0.00020494150898971216, "spike_contrast": 0.6812747128050888, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013962146767880766, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0003699988592416048, 0.0003473494143690914, 0.0003253649629186839, 0.0003058057918678969, 0.0002878782688640058, 0.0002679917961359024, 0.0002544210874475539, 0.00024152429250534624, 0.00022916354646440595, 0.000224049566895701, 0.00020975990628357977, 0.00019981748482678086, 0.000197441695490852, 0.0001882987271528691, 0.00017711948021315038, 0.00017049472080543637, 0.00016402573965024203, 0.00015721219824627042, 0.00015173754945863038, 0.00014811068831477314, 0.00014414411270990968, 0.0001423153153154999, 0.0001392876438330859, 0.00013716594548895955, 0.00013519432104658335, 0.00013375876005738974, 0.00013254763325676322, 0.0001317118585575372, 0.00013049328117631376, 0.00012993274140171707, 0.00012922976748086512, 0.00012818092363886535 ] } } }, "post_mlp_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "7fd9c911e93175896ad618842c120e77b3f017c17b7eec9c2bfb01fc6dbc8ac2", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.0034885562490671873, 0.004831995815038681, 0.005952076520770788, 0.006925336085259914, 0.00385080324485898, 0.005542940925806761, 0.006729288958013058, 0.007927131839096546, 0.004081308841705322, 0.005811207927763462, 0.007534808944910765, 0.008581922389566898, 0.00499920267611742, 0.007102519739419222, 0.008542952127754688, 0.010447698645293713, 0.005041283555328846, 0.007636935915797949, 0.009387043304741383, 0.011020516976714134, 0.005807018838822842, 0.008367395959794521, 0.01024632342159748, 0.012378672137856483, 0.006453148555010557, 0.009127611294388771, 0.011037957854568958, 0.012611051090061665, 0.007010402157902718, 0.009794782847166061, 0.012445803731679916, 0.014087481424212456 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.00015858902861509705, "first_to_last_ratio": 3.757869257587208, "imbalance_abs_log_ratio": 1.3238521099622325, "last_quartile_mean": 4.220184837322449e-05, "mean": 8.57325238712292e-05, "normalized": [ 2.5462257183789148, 2.3032637538343064, 2.0924289533379263, 1.8886492391629932, 1.7031256747388024, 1.5374176267930122, 1.4224231993460965, 1.3049597038569594, 1.2055618292025094, 1.169245405308266, 1.0769812048488787, 1.0144334047898542, 0.9913512297333369, 0.9317090592952362, 0.8577499774884012, 0.81555804918997, 0.7682725064221458, 0.7217807351250844, 0.6783324192661732, 0.6511800118936032, 0.6192916242842709, 0.6053728292385717, 0.5848720876072855, 0.5718124973482859, 0.5462302773879167, 0.5275173250132701, 0.5112998770983732, 0.49691833866078555, 0.47721658788927634, 0.46551989467389665, 0.46012635628943105, 0.45317260249616614 ], "peak_layer": 1, "peak_normalized": 2.5462257183789148, "population_cv": 0.5686111305901927, "rest_mean": 9.231303704803361e-05, "spike_contrast": 0.5437775023084314, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.019775271648541e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00021829435718245804, 0.00019746461475733668, 0.0001793892151908949, 0.0001619186659809202, 0.00014601326256524771, 0.00013180669338908046, 0.00012194793089292943, 0.00011187748896190897, 0.00010335585830034688, 0.00010024235962191597, 9.233231685357168e-05, 8.69699360919185e-05, 8.499104296788573e-05, 7.987776916706935e-05, 7.353707042057067e-05, 6.991984992055222e-05, 6.586594099644572e-05, 6.188008410390466e-05, 5.8155150327365845e-05, 5.582730591413565e-05, 5.309343396220356e-05, 5.190014053368941e-05, 5.014256021240726e-05, 4.90229285787791e-05, 4.682970029534772e-05, 4.522539165918715e-05, 4.383502891869284e-05, 4.2602063331287354e-05, 4.0912982512963936e-05, 3.991019548266195e-05, 3.944779382436536e-05, 3.885163096128963e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.00021242368711682502, "first_to_last_ratio": 1.8345658968465297, "imbalance_abs_log_ratio": 0.6068078850437921, "last_quartile_mean": 0.0001157896194854402, "mean": 0.0001511171253696375, "normalized": [ 1.7717451736709768, 1.636674793183293, 1.5242755762025908, 1.4186515254985965, 1.3264266232459345, 1.2453738409548603, 1.1893512719369177, 1.1330135273991528, 1.085860457802012, 1.0628542818802846, 1.023252039669589, 0.9942070699734702, 0.9784308644775869, 0.9531083940389613, 0.9232390376562555, 0.9040072643238835, 0.8853775311228679, 0.8680181295617843, 0.8515367513400655, 0.8395795332993111, 0.8267439067367663, 0.8185133286645813, 0.8091730400239778, 0.800791325997512, 0.7909899739933552, 0.7821072339266707, 0.7753403490721149, 0.7682422073314642, 0.7604971517068638, 0.7549863467424895, 0.7508596466770359, 0.7467718018887739 ], "peak_layer": 1, "peak_normalized": 1.7717451736709768, "population_cv": 0.2714186609524326, "rest_mean": 0.00015645541295978345, "spike_contrast": 0.7816308178118705, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012229037238284946, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0002677410375326872, 0.0002473295899108052, 0.00023034414334688336, 0.00021438254043459892, 0.00020044577831868082, 0.00018819731485564262, 0.00017973134526982903, 0.000171217747265473, 0.0001640921109355986, 0.00016061548376455903, 0.00015463090676348656, 0.00015024171443656087, 0.0001478576596127823, 0.00014403100067283958, 0.00013951722939964384, 0.00013661097909789532, 0.00013379570737015456, 0.0001311724045081064, 0.0001286817860091105, 0.00012687484559137374, 0.0001249351626029238, 0.00012369138130452484, 0.00012227990373503417, 0.00012101328320568427, 0.00011953213106608018, 0.0001181897969217971, 0.0001171672047348693, 0.00011609455395955592, 0.00011492414341773838, 0.0001140913664130494, 0.00011346775136189535, 0.00011285000800853595 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.18837944976985455, "first_to_last_ratio": 1.8345658800697515, "imbalance_abs_log_ratio": 0.6068078758989703, "last_quartile_mean": 0.10268339328467846, "mean": 0.1340121782850474, "normalized": [ 1.7717451751532867, 1.6366747548098712, 1.5242756140248972, 1.4186514377480737, 1.3264266357207033, 1.2453739091726768, 1.1893513017871604, 1.1330134634090196, 1.0858604665419038, 1.0628542749024097, 1.0232520332438115, 0.9942070832344971, 0.9784308561813967, 0.9531084116998004, 0.9232390798282929, 0.9040072094634526, 0.8853775581660035, 0.8680181154203164, 0.8515367049614931, 0.8395794986042229, 0.8267439263823189, 0.8185133396785964, 0.809173050926764, 0.8007913534932437, 0.7909900046670136, 0.782107217826699, 0.7753403703776178, 0.7682422246066585, 0.7604971603471317, 0.7549863443945886, 0.7508596538022533, 0.7467717694238248 ], "peak_layer": 1, "peak_normalized": 1.7717451751532867, "population_cv": 0.27141865758333467, "rest_mean": 0.13874622461972413, "spike_contrast": 0.7816308398662988, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.10844832807779312, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.23743543028831482, 0.21933434903621674, 0.20427149534225464, 0.19011656939983368, 0.17775732278823853, 0.16689527034759521, 0.15938755869865417, 0.15183760225772858, 0.14551852643489838, 0.14243541657924652, 0.13712823390960693, 0.1332358568906784, 0.1311216503381729, 0.12772813439369202, 0.1237252801656723, 0.12114797532558441, 0.1186513751745224, 0.1163249984383583, 0.11411628872156143, 0.11251387745141983, 0.11079375445842743, 0.10969075560569763, 0.10843904316425323, 0.107315793633461, 0.10600229352712631, 0.104811891913414, 0.10390505194664001, 0.10295381397008896, 0.1019158810377121, 0.10117736458778381, 0.10062433779239655, 0.10007651150226593 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 6.873207303215167e-05, "first_to_last_ratio": 2.248757814140914, "imbalance_abs_log_ratio": 0.8103779811585758, "last_quartile_mean": 3.0564462122129044e-05, "mean": 4.6857780716891284e-05, "normalized": [ 1.7400119939228862, 1.6405409364325143, 1.555121247708426, 1.4944054279879166, 1.4327475398601994, 1.3414100714975594, 1.2804101716466747, 1.2499370459040815, 1.192804641985012, 1.1608858830017554, 1.098191983597808, 1.057744066133373, 1.0542717513613309, 1.021869458338179, 0.9775749297615711, 0.9570927445976433, 0.9210091333285576, 0.8815125232807631, 0.847820728512489, 0.8204970180441978, 0.7871838673851995, 0.7759041238100245, 0.7506938949785397, 0.7421070509174572, 0.7072987335050381, 0.6891811912847667, 0.6695407652827761, 0.6616333377893114, 0.6345027924930273, 0.6212629426015038, 0.6200302724102036, 0.614801730639213 ], "peak_layer": 1, "peak_normalized": 1.7400119939228862, "population_cv": 0.32175451551289735, "rest_mean": 4.9004235565748616e-05, "spike_contrast": 0.7196709452950106, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.5266924533061685e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 8.153310045599937e-05, 7.687210745643824e-05, 7.28695304132998e-05, 7.002452184678987e-05, 6.713537004543468e-05, 6.28554989816621e-05, 5.9997179050697014e-05, 5.8569276006892323e-05, 5.589217835222371e-05, 5.439653614303097e-05, 5.1458839152473956e-05, 4.9563539505470544e-05, 4.9400834541302174e-05, 4.788253500009887e-05, 4.58069916930981e-05, 4.4847241952084005e-05, 4.315644400776364e-05, 4.130572051508352e-05, 3.972699778387323e-05, 3.8446669350378215e-05, 3.688568904181011e-05, 3.635714529082179e-05, 3.5175849916413426e-05, 3.477348946034908e-05, 3.3142448955914006e-05, 3.2293501135427505e-05, 3.13731943606399e-05, 3.1002669857116416e-05, 2.9731392714893445e-05, 2.9111002731951885e-05, 2.905324254243169e-05, 2.8808244678657502e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 0.00017190765356644988, "first_to_last_ratio": 1.4894621368061003, "imbalance_abs_log_ratio": 0.3984150727783497, "last_quartile_mean": 0.00011541592721187044, "mean": 0.00013894661265112518, "normalized": [ 1.3739040544627474, 1.3296102750467884, 1.2883342981472405, 1.2499918159027226, 1.2130627732820631, 1.175544413473833, 1.1474083142393565, 1.1199112790671786, 1.092909618037032, 1.0755929927697745, 1.0512861461262244, 1.0312571117882559, 1.0194388253067554, 1.0019061415077835, 0.980344379776871, 0.9647961249933756, 0.9493075664595542, 0.9344817410273759, 0.9198929201794295, 0.9079992279088673, 0.8952920956888009, 0.8869530532606651, 0.8771902552778059, 0.8683889858357474, 0.8579407806493952, 0.8482823463732029, 0.8408658204630546, 0.8330843095877946, 0.8243382853592501, 0.818115422616924, 0.8135611224181157, 0.8090075029660142 ], "peak_layer": 1, "peak_normalized": 1.3739040544627474, "population_cv": 0.16161403917150835, "rest_mean": 0.00014210757001786045, "spike_contrast": 0.8576421569620596, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012187744287075474, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00019089931447524577, 0.00018474484386388212, 0.00017900968668982387, 0.00017368212866131216, 0.0001685509632807225, 0.00016333791427314281, 0.00015942849859129637, 0.00015560787869617343, 0.00015185608936008066, 0.00014945000293664634, 0.00014607264893129468, 0.0001432896824553609, 0.00014164757158141583, 0.0001392114645568654, 0.00013621553080156446, 0.0001340551534667611, 0.00013190307072363794, 0.00012984307250007987, 0.0001278160052606836, 0.00012616341700777411, 0.0001243978040292859, 0.00012323912233114243, 0.00012188261462142691, 0.00012065970804542303, 0.00011920796532649547, 0.00011786595860030502, 0.00011683545744745061, 0.00011575424287002534, 0.00011453901242930442, 0.00011367436673026532, 0.00011304156214464456, 0.00011240885214647278 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 0.00014552358334185556, "first_to_last_ratio": 1.262638843036328, "imbalance_abs_log_ratio": 0.2332038508028458, "last_quartile_mean": 0.00011525352965691127, "mean": 0.00012953498708156985, "normalized": [ 1.1676304712442283, 1.1525836984023852, 1.1391521450612143, 1.1288261090295841, 1.1167600441451297, 1.1054423860149858, 1.094731137320412, 1.082319740349084, 1.067957453000188, 1.0563935463552665, 1.04560972624621, 1.0337959762176818, 1.0235535208863231, 1.0159040893770892, 1.0050628637064216, 0.9950792389307415, 0.9865814183778941, 0.9786230528286792, 0.9704418054120744, 0.9608381133357455, 0.9508407831226554, 0.9429749854457838, 0.9346839821902475, 0.9262276149685831, 0.9170542960948607, 0.9059488475652709, 0.8984648923618161, 0.8908037775288885, 0.8831483920181569, 0.8778579808382188, 0.8744051093280881, 0.8703028022960919 ], "peak_layer": 1, "peak_normalized": 1.1676304712442283, "population_cv": 0.0896642712448568, "rest_mean": 0.00013110964479252557, "spike_contrast": 0.9231344927669946, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012103183544240892, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00015124899800866842, 0.00014929991448298097, 0.00014756005839444697, 0.00014622247545048594, 0.00014465949789155275, 0.00014319346519187093, 0.00014180598373059183, 0.00014019827358424664, 0.00013833785487804562, 0.00013683992438018322, 0.0001354430423816666, 0.00013391274842433631, 0.0001325859921053052, 0.00013159512309357524, 0.00013019080506637692, 0.00012889757636003196, 0.00012779681128449738, 0.00012676592450588942, 0.00012570616672746837, 0.00012446215259842575, 0.00012316714855842292, 0.00012214825255796313, 0.00012107427755836397, 0.00011997888213954866, 0.00011879061639774591, 0.00011735207226593047, 0.0001163826382253319, 0.0001153902558144182, 0.00011439861555118114, 0.00011371332220733166, 0.00011326605454087257, 0.0001127346622524783 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00029742828155576717, "first_to_last_ratio": 2.2781890121807993, "imbalance_abs_log_ratio": 0.8233808344454435, "last_quartile_mean": 0.00013055469935352448, "mean": 0.00019329176166138495, "normalized": [ 1.8891850069388627, 1.7801603926271052, 1.6773544463608039, 1.566500262900107, 1.476456430986117, 1.3754925511314788, 1.3084504582544147, 1.236424592515421, 1.172965600373877, 1.150871569148191, 1.0802294364746856, 1.0319209662753672, 1.0080063247818516, 0.9698985570693867, 0.9067931019504507, 0.8756163462587222, 0.8444435806568995, 0.8113939671049883, 0.7809677247391495, 0.7613442367276676, 0.7402582678385915, 0.7293562882295931, 0.7162426695419845, 0.7062419232587185, 0.6940340549797547, 0.6866100033591334, 0.6831688143117768, 0.6753320518823956, 0.6715919825156708, 0.6686410475337363, 0.6640239865195754, 0.6600233567535221 ], "peak_layer": 1, "peak_normalized": 1.8891850069388627, "population_cv": 0.357915029863326, "rest_mean": 0.00020341356920888993, "spike_contrast": 0.6815376252628051, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001386340009048581, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00036516389809548855, 0.0003440903383307159, 0.0003242187958676368, 0.0003027915954589844, 0.00028538686456158757, 0.00026587137836031616, 0.0002529126941226423, 0.00023899068764876574, 0.00022672458726447076, 0.00022245399304665625, 0.0002087994507746771, 0.00019946182146668434, 0.00019483931828290224, 0.00018747340072877705, 0.00017527563613839447, 0.00016924942610785365, 0.00016322398732881993, 0.00015683576930314302, 0.0001509546273155138, 0.00014716156874783337, 0.0001430858246749267, 0.0001409785618307069, 0.00013844380737282336, 0.00013651074550580233, 0.00013415106514003128, 0.00013271605712361634, 0.00013205090363044292, 0.00013053612201474607, 0.00012981319741811603, 0.00012924280599690974, 0.00012835036613978446, 0.00012757707736454904 ] } } }, "pre_attention_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "de71d4cf2d509a1063f6eac7f50c4885e9a058dee7e4b993ad40f5527f528d00", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.02802751213312149, 0.01413465291261673, 0.014225807972252369, 0.01433455292135477, 0.01440478302538395, 0.009703479707241058, 0.00977881159633398, 0.009858808480203152, 0.009950309991836548, 0.00752287870272994, 0.007571155205368996, 0.007693085819482803, 0.007788818795233965, 0.006353048607707024, 0.006389216054230928, 0.0064707002602517605, 0.00656962301582098, 0.005553483963012695, 0.0056249531917274, 0.005688921548426151, 0.005768724251538515, 0.004966245498508215, 0.005057469941675663, 0.005098348017781973, 0.005216081161051989, 0.0046758754178881645, 0.004727107938379049, 0.004777040798217058, 0.004837613552808762, 0.0044186655431985855, 0.004510571248829365, 0.0045849340967834 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 0.0001580166108396952, "first_to_last_ratio": 1.9713740353643698, "imbalance_abs_log_ratio": 0.6787307795042399, "last_quartile_mean": 8.015557068574708e-05, "mean": 0.00011843875881822896, "normalized": [ 0.8918679597819287, 1.5814819593588525, 1.5722239987443252, 1.3246067564285042, 1.1252650527356378, 1.512743589128775, 1.3788112428745087, 1.2863035124436846, 1.097095055843533, 1.468154353173571, 1.1523254606068964, 1.0839888571909224, 1.054518217028451, 1.146335080549274, 1.1665208527784754, 0.8931136212023528, 0.8203804970547371, 1.073741223277434, 0.919654742394771, 0.8223971330190447, 0.7711288002244273, 0.8470775332584999, 0.8298711629738247, 0.7662487463744663, 0.7210382462310316, 0.7071376059209309, 0.7189559392079661, 0.727980518763353, 0.6111916361855597, 0.6986829946379066, 0.6244678178312321, 0.6046898327751223 ], "peak_layer": 2, "peak_normalized": 1.5814819593588525, "population_cv": 0.2904715201464359, "rest_mean": 0.00012310891098523926, "spike_contrast": 0.7572151875143335, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.321993711637333e-05, "top_five_layers": [ 2, 3, 6, 10, 7 ], "top_five_spike_overlap": 0 }, "values": [ 0.00010563173418631777, 0.0001873087603598833, 0.00018621225899551064, 0.00015688478015363216, 0.0001332749961875379, 0.00017916747310664505, 0.00016330469225067645, 0.00015234819147735834, 0.00012993857671972364, 0.00017388637934345752, 0.0001364799973089248, 0.0001283862948184833, 0.00012489582877606153, 0.00013577050413005054, 0.00013816128193866462, 0.00010577926877886057, 9.71648478298448e-05, 0.00012717257777694613, 0.00010892276623053476, 9.74036956904456e-05, 9.133153798757121e-05, 0.0001003268116619438, 9.828891052166e-05, 9.07535504666157e-05, 8.539887494407594e-05, 8.375250035896897e-05, 8.515224908478558e-05, 8.622110908618197e-05, 7.238877878990024e-05, 8.275114669231698e-05, 7.396119326585904e-05, 7.161871326388791e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.0001664285200604354, "first_to_last_ratio": 1.9719337364651075, "imbalance_abs_log_ratio": 0.6790146534176558, "last_quartile_mean": 8.439863722742302e-05, "mean": 0.0001245867656507471, "normalized": [ 0.8984334461398115, 1.5927952483854981, 1.5658595485525297, 1.3198735865176374, 1.131213894321221, 1.5125315092450449, 1.3796810696180641, 1.286366052942558, 1.0960606282761096, 1.4536884557404346, 1.1543342746982923, 1.0797559630746076, 1.0497530570909623, 1.1457182987306291, 1.1551962694199982, 0.9023721663305979, 0.8238710779144349, 1.0632503993742801, 0.9211624832492573, 0.8274335806257342, 0.7739032412019898, 0.850990856837025, 0.8281317613098801, 0.7681943949939335, 0.7208251570035091, 0.7089546837555952, 0.7203162530736348, 0.7236174125342815, 0.6145740290254166, 0.6994991973458914, 0.6274381906330746, 0.6042038120380651 ], "peak_layer": 2, "peak_normalized": 1.5927952483854981, "population_cv": 0.2889665774838781, "rest_mean": 0.00012946851159169995, "spike_contrast": 0.7586812913967177, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.822533756960183e-05, "top_five_layers": [ 2, 3, 6, 10, 7 ], "top_five_spike_overlap": 0 }, "values": [ 0.00011193291720701382, 0.0001984412083402276, 0.0001950853766174987, 0.000164438781212084, 0.00014093428035266697, 0.00018844140868168324, 0.00017189000209327787, 0.00016026418597903103, 0.00013655464863404632, 0.00018111034296453, 0.00014381477376446128, 0.0001345233031315729, 0.00013078533811494708, 0.00014274133718572557, 0.00014392216689884663, 0.00011242362961638719, 0.00010264343291055411, 0.0001324669283349067, 0.00011476465442683548, 0.00010308727360097691, 9.641810174798593e-05, 0.00010602219845168293, 0.00010317425767425448, 9.570685506332666e-05, 8.980527491075918e-05, 8.832637104205787e-05, 8.974187221610919e-05, 9.015315299620852e-05, 7.656779052922502e-05, 8.714834257261828e-05, 7.817049481673166e-05, 7.527579873567447e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.14759047608822584, "first_to_last_ratio": 1.9719337947467823, "imbalance_abs_log_ratio": 0.6790146829732503, "last_quartile_mean": 0.07484555337578058, "mean": 0.1104847895912826, "normalized": [ 0.8984334974418968, 1.5927952138130326, 1.565859646322909, 1.3198735575362357, 1.1312139236939245, 1.5125315007911495, 1.3796810264894532, 1.286366010451999, 1.0960606168617568, 1.4536884922319049, 1.1543342013148132, 1.0797558953237962, 1.0497530983161607, 1.1457182562234747, 1.1551962948455465, 0.9023721940039697, 0.8238711061937815, 1.0632504861804335, 0.9211625151546727, 0.8274335809310764, 0.7739032642161865, 0.8509909070635955, 0.8281317391228538, 0.7681943896830757, 0.7208251689669645, 0.7089546589522648, 0.7203162343708721, 0.7236173971219663, 0.6145740294939214, 0.6994991437375198, 0.6274381319743458, 0.6042038211744466 ], "peak_layer": 2, "peak_normalized": 1.5927952138130326, "population_cv": 0.28896658018674404, "rest_mean": 0.11481397047086998, "spike_contrast": 0.7586813040631772, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.08710721284151077, "top_five_layers": [ 2, 3, 6, 10, 7 ], "top_five_spike_overlap": 0 }, "values": [ 0.09926323592662811, 0.1759796440601349, 0.1730036735534668, 0.14582595229148865, 0.12498193234205246, 0.16711172461509705, 0.15243376791477203, 0.14212387800216675, 0.12109802663326263, 0.16061046719551086, 0.1275363713502884, 0.1192966029047966, 0.11598175019025803, 0.12658444046974182, 0.12763161957263947, 0.09969840198755264, 0.0910252258181572, 0.11747300624847412, 0.10177444666624069, 0.09141882508993149, 0.08550453931093216, 0.09402155131101608, 0.09149596095085144, 0.08487379550933838, 0.0796402171254158, 0.0783287063241005, 0.07958398759365082, 0.07994871586561203, 0.06790108233690262, 0.07728401571512222, 0.069322369992733, 0.06675533205270767 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 5.093040158499207e-05, "first_to_last_ratio": 0.8860093039525244, "imbalance_abs_log_ratio": 0.12102782735710801, "last_quartile_mean": 5.748291960117058e-05, "mean": 5.7093861016710434e-05, "normalized": [ 0.4337136137178416, 0.9185569667519601, 0.9444095926066538, 0.8728787927927439, 0.7849140301431445, 1.1296292591692207, 1.0726593944235638, 0.9796135302182263, 0.9386642177229508, 1.3046353648934683, 1.0461832118506482, 1.0032917361864164, 0.9662905330123907, 1.0760080966276753, 1.1828739682986207, 0.9087955578115051, 0.9194941496879511, 1.198292373503509, 1.0946132296674007, 1.0673695485587213, 0.9244260207272295, 1.0525679458300632, 1.125653431968099, 0.9999504891362015, 0.9629027708994021, 1.0063234987710272, 0.9975933861195996, 1.0791525148711678, 0.9412468232151083, 1.085173475571988, 1.0031556955576197, 0.9789667796878816 ], "peak_layer": 10, "peak_normalized": 1.3046353648934683, "population_cv": 0.1441687540611157, "rest_mean": 5.695535414647397e-05, "spike_contrast": 1.0155638391297448, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.784179811598733e-05, "top_five_layers": [ 10, 18, 15, 6, 23 ], "top_five_spike_overlap": 1 }, "values": [ 2.4762384782661684e-05, 5.2443963795667514e-05, 5.3919990023132414e-05, 4.9836020480142906e-05, 4.481377254705876e-05, 6.449489592341706e-05, 6.124226638348773e-05, 5.592991874436848e-05, 5.3591964388033375e-05, 7.448667020071298e-05, 5.973063889541663e-05, 5.728179894504137e-05, 5.516925739357248e-05, 6.143345672171563e-05, 6.75348419463262e-05, 5.1886647270293906e-05, 5.249747118796222e-05, 6.84151382301934e-05, 6.249569560168311e-05, 6.094024865888059e-05, 5.277905074763112e-05, 6.009516800986603e-05, 6.426790059776977e-05, 5.7091034250333905e-05, 5.4975836974335834e-05, 5.74548939766828e-05, 5.695645813830197e-05, 6.161298369988799e-05, 5.373941530706361e-05, 6.19567435933277e-05, 5.727403186028823e-05, 5.58929932594765e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 5.801741053801379e-05, "first_to_last_ratio": 0.954195857595535, "imbalance_abs_log_ratio": 0.046886327142486896, "last_quartile_mean": 6.08024129178375e-05, "mean": 6.296396202287724e-05, "normalized": [ 0.4796364558374551, 0.9568353419130547, 0.9804638447804884, 0.8949545354119591, 0.8237377234766136, 1.1597846916175047, 1.085649359633547, 0.9904450076993521, 0.9595078978373608, 1.2941748206155441, 1.0680753760655664, 1.0231559828609242, 0.9904019047543832, 1.096490963646882, 1.1850351594596897, 0.9422215946487361, 0.9418631353047864, 1.1705024179981143, 1.0926353878857977, 1.0382275689133567, 0.9320833313178482, 1.0784583323883348, 1.0985080196909318, 0.9917906747078287, 0.9476388143731139, 0.9855066582268529, 0.978396290111904, 1.028407839812513, 0.8982055143685478, 1.0282804954549911, 0.9451124964564701, 0.9138123627295472 ], "peak_layer": 10, "peak_normalized": 1.2941748206155441, "population_cv": 0.13431233496292436, "rest_mean": 6.285090866084729e-05, "spike_contrast": 1.0115120295379672, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.357445017783903e-05, "top_five_layers": [ 10, 15, 18, 6, 23 ], "top_five_spike_overlap": 1 }, "values": [ 3.019981159013696e-05, 6.0246144130360335e-05, 6.173388828756288e-05, 5.634988337988034e-05, 5.186579073779285e-05, 7.302463927771896e-05, 6.835678505012766e-05, 6.236234185053036e-05, 6.041441884008236e-05, 8.148637425620109e-05, 6.725025741616264e-05, 6.442195444833487e-05, 6.235962791834027e-05, 6.903941539349034e-05, 7.461450877599418e-05, 5.9326004702597857e-05, 5.930343468207866e-05, 7.369946979451925e-05, 6.879665306769311e-05, 6.537092122016475e-05, 5.8687659475253895e-05, 6.790400948375463e-05, 6.916641723364592e-05, 6.244707037694752e-05, 5.966709431959316e-05, 6.205140380188823e-05, 6.160370685392991e-05, 6.475263216998428e-05, 5.655457789544016e-05, 6.474461406469345e-05, 5.950802733423188e-05, 5.753724690293893e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 1.7888660295284353e-05, "first_to_last_ratio": 0.41855462748958633, "imbalance_abs_log_ratio": 0.8709478659353102, "last_quartile_mean": 4.2739129185065394e-05, "mean": 3.1986895351110434e-05, "normalized": [ 0.2593955501951102, 0.4997829486650695, 0.48480994473028793, 0.4968760363401999, 0.5474125689219601, 0.7788406177150305, 0.7554473604850604, 0.6514322393717875, 0.7578135841604356, 0.9674810188795085, 0.9032772556875033, 0.9019220651946442, 0.8311356963419073, 0.9587970177975769, 1.1401394869948454, 0.8953886492354074, 1.0262272842074007, 1.1413916918814533, 1.174037958462901, 1.194927715786966, 1.0267140632187253, 1.3434898267078448, 1.3729067467361273, 1.2011929483563188, 1.1735959904166378, 1.3266745689353452, 1.2739952988364465, 1.4049535244931144, 1.2752889026856997, 1.4680979746684197, 1.4182292846928506, 1.3483241791974143 ], "peak_layer": 30, "peak_normalized": 1.4680979746684197, "population_cv": 0.3184955307507873, "rest_mean": 3.066251991657299e-05, "spike_contrast": 1.276428773763581, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.913852269761264e-05, "top_five_layers": [ 30, 31, 28, 23, 32 ], "top_five_spike_overlap": 1 }, "values": [ 8.297258318634704e-06, 1.5986504877218977e-05, 1.5507564967265353e-05, 1.589352177688852e-05, 1.7510028555989265e-05, 2.491269333404489e-05, 2.416441566310823e-05, 2.083729486912489e-05, 2.4240103812189773e-05, 3.094671410508454e-05, 2.889303505071439e-05, 2.8849686714238487e-05, 2.6585450541460887e-05, 3.066893987124786e-05, 3.6469522456172854e-05, 2.8640703021665104e-05, 3.282582474639639e-05, 3.6509576602838933e-05, 3.755382931558415e-05, 3.822202779701911e-05, 3.2841395295690745e-05, 4.2974068492185324e-05, 4.391502443468198e-05, 3.842243313556537e-05, 3.7539692129939795e-05, 4.243620060151443e-05, 4.0751154301688075e-05, 4.494010136113502e-05, 4.0792532672639936e-05, 4.695989628089592e-05, 4.5364751713350415e-05, 4.3128704419359565e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00022091567916504573, "first_to_last_ratio": 1.232986382535498, "imbalance_abs_log_ratio": 0.20943917994887448, "last_quartile_mean": 0.00017917122386279516, "mean": 0.0002198539991695725, "normalized": [ 0.4672441326905094, 1.0299890566632133, 1.08986441571188, 1.0084939820045402, 0.8814237539069011, 1.2693826047962031, 1.1886514905905403, 1.1035827467935222, 1.0473307057033197, 1.4274044616309502, 1.1442998336482333, 1.0911283607583675, 1.1094038041384326, 1.1946873974128753, 1.2511466100570947, 0.9923165311956169, 0.9239703707694655, 1.2473578196113966, 1.1181336030529359, 1.0160745414007064, 0.9397394354798955, 0.9905446518023159, 1.0121913654482901, 0.9359943959598221, 0.8839446941554127, 0.8510229499852975, 0.8648513533654478, 0.8786898836618169, 0.7455559493639876, 0.8399702482950041, 0.7392957942269532, 0.7163130557190521 ], "peak_layer": 10, "peak_normalized": 1.4274044616309502, "population_cv": 0.18952709251819305, "rest_mean": 0.0002217885959436427, "spike_contrast": 0.9441746799407325, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00020940717658959329, "top_five_layers": [ 10, 6, 15, 18, 14 ], "top_five_spike_overlap": 0 }, "values": [ 0.00010272549116052687, 0.00022644721320830286, 0.00023961105034686625, 0.00022172143508214504, 0.0001937845372594893, 0.0002790788421407342, 0.0002613297838252038, 0.00024262708029709756, 0.00023025984410196543, 0.000313820579322055, 0.00025157889467664063, 0.00023988893372006714, 0.00024390686303377151, 0.000262656802078709, 0.00027506958576850593, 0.0002181647578254342, 0.00020313858112785965, 0.00027423660503700376, 0.0002458261442370713, 0.00022338805138133466, 0.00020660547306761146, 0.00021777520305477083, 0.00022253431961871684, 0.00020578211115207523, 0.0001943387760547921, 0.00018710079893935472, 0.0001901410287246108, 0.00019318348495289683, 0.00016391345707233995, 0.00018467081827111542, 0.00016253713693004102, 0.00015748428995721042 ] } } }, "pre_mlp_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "2999a257265024350fd2665e2a8d452a03cd662193b4245d15290408ad29922d", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.014061377383768559, 0.014165290631353855, 0.014281988143920898, 0.014350161887705326, 0.00966146495193243, 0.009741888381540775, 0.009820535778999329, 0.00991273857653141, 0.0074927168898284435, 0.007546746172010899, 0.00766373984515667, 0.007762390188872814, 0.006325984839349985, 0.006375366821885109, 0.006445344537496567, 0.006555322092026472, 0.005531130358576775, 0.005611324217170477, 0.0056777093559503555, 0.00574630219489336, 0.004946626722812653, 0.0050392793491482735, 0.005084683652967215, 0.005203457549214363, 0.004667261149734259, 0.004717131145298481, 0.004768412560224533, 0.004826710093766451, 0.0044052014127373695, 0.004504534415900707, 0.004574235528707504, 0.004644096828997135 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 3.729895206561196e-05, "first_to_last_ratio": 1.2469721805352876, "imbalance_abs_log_ratio": 0.22071835733540474, "last_quartile_mean": 2.9911615229138988e-05, "mean": 3.340667609563752e-05, "normalized": [ 1.308752778077389, 1.2456464596610868, 1.0236956931411572, 0.9922285647386262, 1.26903181062375, 1.1170846373362797, 1.0287316533628064, 0.9469237624012169, 1.1300261751959062, 1.1132698789554223, 1.0126240729566007, 0.9379202574439188, 1.1130503370514597, 1.070713488608195, 0.9897642722064988, 0.922085688720865, 1.036831181462568, 0.9898105546019274, 0.9395130252451869, 0.9001967292772246, 0.9975952535130206, 0.9404117749145336, 0.9224862220158919, 0.8885787590271026, 0.9461670180110204, 0.8974372095121397, 0.8904691498559152, 0.8684437934704528, 0.9244147277585869, 0.9015689206268677, 0.8740901912628669, 0.8604359589635168 ], "peak_layer": 1, "peak_normalized": 1.308752778077389, "population_cv": 0.1164905978879517, "rest_mean": 3.378375206213807e-05, "spike_contrast": 0.9285666618330386, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.137046587653458e-05, "top_five_layers": [ 1, 5, 2, 9, 6 ], "top_five_spike_overlap": 0 }, "values": [ 4.3721080146497115e-05, 4.161290780757554e-05, 3.419827044126578e-05, 3.314705827506259e-05, 4.239413465256803e-05, 3.73180846509058e-05, 3.4366505133220926e-05, 3.1633575417799875e-05, 3.7750418414361775e-05, 3.719064625329338e-05, 3.3828404411906376e-05, 3.1332798243965954e-05, 3.718331208801828e-05, 3.5768978705164045e-05, 3.306473445263691e-05, 3.080381793552078e-05, 3.463708344497718e-05, 3.3066280593629926e-05, 3.138600732199848e-05, 3.007258055731654e-05, 3.332634150865488e-05, 3.1416031561093405e-05, 3.0817198421573266e-05, 2.9684462788281962e-05, 3.1608295103069395e-05, 2.9980394174344838e-05, 2.9747614462394267e-05, 2.9011820515734144e-05, 3.0881623388268054e-05, 3.01184209092753e-05, 2.9200447897892445e-05, 2.874430538213346e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 6.705871965095866e-05, "first_to_last_ratio": 0.7180297289018083, "imbalance_abs_log_ratio": 0.33124430563536084, "last_quartile_mean": 9.33926785364747e-05, "mean": 8.228966771639534e-05, "normalized": [ 0.8685826408392191, 0.8249006353131848, 0.7154905339803773, 0.6884730939760333, 0.9470487095509537, 0.8638069620260335, 0.8256155016985178, 0.7853664463613442, 0.9851686390305121, 0.9692511243547932, 0.9083804947955202, 0.8729855457896263, 1.0422548534804916, 1.0255492616133721, 0.9871528463964067, 0.9398223255303129, 1.0958375595009802, 1.0726325633807163, 1.0495259774381736, 1.0230838788771643, 1.1576441025653552, 1.1142532603054807, 1.1008567438213033, 1.056908774400763, 1.159063578749664, 1.1192434432436902, 1.1032241584740294, 1.1060435702772593, 1.1786235078649367, 1.1628365878275773, 1.141840825041133, 1.1085318534950752 ], "peak_layer": 29, "peak_normalized": 1.1786235078649367, "population_cv": 0.1341844256762087, "rest_mean": 8.049536754023629e-05, "spike_contrast": 1.1426606459269575, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.197888866765424e-05, "top_five_layers": [ 29, 30, 25, 21, 31 ], "top_five_spike_overlap": 2 }, "values": [ 7.14753768988885e-05, 6.788079917896539e-05, 5.887747829547152e-05, 5.66542221349664e-05, 7.793232362018898e-05, 7.108238787623122e-05, 6.793962529627606e-05, 6.462754390668124e-05, 8.106919995043427e-05, 7.975935295689851e-05, 7.475032907677814e-05, 7.183769048424438e-05, 8.576680556870997e-05, 8.4392107964959e-05, 8.123247971525416e-05, 7.733766688033938e-05, 9.017610864248127e-05, 8.826657722238451e-05, 8.636514394311234e-05, 8.418923243880272e-05, 9.526214853394777e-05, 9.169153054244816e-05, 9.0589135652408e-05, 8.697267185198143e-05, 9.537895675748587e-05, 9.210217103827745e-05, 9.078394941752777e-05, 9.101595787797123e-05, 9.698853682493791e-05, 9.56894364207983e-05, 9.396170207764953e-05, 9.122071787714958e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.05946834245696664, "first_to_last_ratio": 0.7180297222365345, "imbalance_abs_log_ratio": 0.33124431491808765, "last_quartile_mean": 0.08282156102359295, "mean": 0.07297530013602227, "normalized": [ 0.8685826122359729, 0.8249005917343738, 0.71549052911611, 0.6884731350635278, 0.9470486699630107, 0.8638070113577097, 0.8256154769796886, 0.7853664269052657, 0.9851686341386658, 0.9692511556795396, 0.9083805454214465, 0.8729855579949275, 1.0422548040385637, 1.0255492365750694, 0.9871528950273681, 0.9398223256645052, 1.0958375053304956, 1.07263252629438, 1.0495259690486791, 1.0230837911446407, 1.1576441435704576, 1.1142533045483842, 1.1008568124470575, 1.0569088284972943, 1.1590636022344982, 1.119243411839159, 1.1032241423789504, 1.1060435591240758, 1.1786235037173105, 1.1628366076958818, 1.141840809884862, 1.1085318743481278 ], "peak_layer": 29, "peak_normalized": 1.1786235037173105, "population_cv": 0.13418442668798974, "rest_mean": 0.07138409617322462, "spike_contrast": 1.1426607032635474, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.08156780153512955, "top_five_layers": [ 29, 30, 25, 21, 31 ], "top_five_spike_overlap": 2 }, "values": [ 0.06338507682085037, 0.0601973682641983, 0.05221313610672951, 0.050241533666849136, 0.0691111609339714, 0.06303657591342926, 0.060249537229537964, 0.05731235072016716, 0.07189297676086426, 0.07073139399290085, 0.06628934293985367, 0.06370638310909271, 0.07605885714292526, 0.0748397633433342, 0.07203777879476547, 0.06858381628990173, 0.07996907085180283, 0.07827568054199219, 0.07658947259187698, 0.07465984672307968, 0.0844794288277626, 0.08131296932697296, 0.0803353562951088, 0.07712823897600174, 0.08458301424980164, 0.08167712390422821, 0.08050811290740967, 0.08071386069059372, 0.0860104039311409, 0.08485835045576096, 0.08332617580890656, 0.08089544624090195 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 7.408341275549901e-06, "first_to_last_ratio": 0.49124437919492936, "imbalance_abs_log_ratio": 0.7108135577065074, "last_quartile_mean": 1.5080765479069669e-05, "mean": 1.1844782648040564e-05, "normalized": [ 0.5657990507718705, 0.5382233871697658, 0.49942145823009426, 0.48906984415931193, 0.8062921756393574, 0.7122953827323945, 0.7013508394120195, 0.6911626993599347, 0.8940675881372303, 0.868722127857376, 0.9735911331661544, 0.854855860776803, 0.989483587379368, 1.0059088486455374, 1.0303729006301217, 0.9941025538400318, 1.1620912657632405, 1.127012997009919, 1.113401270263767, 1.066097770938846, 1.1826034542812638, 1.1643547935214302, 1.2049778184223279, 1.1791491538213539, 1.2365172470752577, 1.219921447624247, 1.2337761202582145, 1.2288278254433027, 1.3039217543779833, 1.3631114810840155, 1.2856635723507386, 1.3138525898567213 ], "peak_layer": 30, "peak_normalized": 1.3631114810840155, "population_cv": 0.2549101540724733, "rest_mean": 1.1420299651271543e-05, "spike_contrast": 1.237882653019464, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.4136990830593277e-05, "top_five_layers": [ 30, 32, 29, 31, 25 ], "top_five_spike_overlap": 1 }, "values": [ 6.701766778860474e-06, 6.3751390371180605e-06, 5.915538622502936e-06, 5.7929260037781205e-06, 9.550355571263935e-06, 8.436983989668079e-06, 8.307348252856173e-06, 8.186671948351432e-06, 1.0590036254143342e-05, 1.0289824786013924e-05, 1.1531975360412616e-05, 1.0125581866304856e-05, 1.1720218026312068e-05, 1.1914771675947122e-05, 1.220454305439489e-05, 1.1774928680097219e-05, 1.3764718460151926e-05, 1.334922399109928e-05, 1.3187996046326589e-05, 1.2627696378331166e-05, 1.4007680874783546e-05, 1.379152945446549e-05, 1.4272700354922563e-05, 1.3966765436634887e-05, 1.4646278032159898e-05, 1.4449704394792207e-05, 1.4613809980801307e-05, 1.4555198504240252e-05, 1.5444669770658948e-05, 1.614575921848882e-05, 1.5228405572997872e-05, 1.5562298358418047e-05 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 5.6029601637419546e-05, "first_to_last_ratio": 0.6070213107493427, "imbalance_abs_log_ratio": 0.49919138022039955, "last_quartile_mean": 9.230252817360451e-05, "mean": 7.840498903988191e-05, "normalized": [ 0.6598133090619259, 0.6419677920829977, 0.6005256373772865, 0.5913463573822231, 0.8562376513989928, 0.8062968559250673, 0.7923156566726156, 0.768439227921366, 0.9790122729268586, 0.9665730330981341, 0.9163710065350301, 0.8881686280668529, 1.0631190324760253, 1.0511030519715439, 1.0188080221730096, 0.9712812246823179, 1.1361272291219966, 1.1138561419734825, 1.0918134348286874, 1.0639217496772315, 1.203414696914147, 1.1592426077267617, 1.1442664103435203, 1.097952968622282, 1.2042281788778202, 1.1625445130141736, 1.1452986211445626, 1.1473932954411665, 1.2233942816523278, 1.205318760675343, 1.1829469858778916, 1.146901364356358 ], "peak_layer": 29, "peak_normalized": 1.2233942816523278, "population_cv": 0.1898597249088972, "rest_mean": 7.605543874820728e-05, "spike_contrast": 1.1977126437531995, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.109256061492487e-05, "top_five_layers": [ 29, 30, 25, 21, 31 ], "top_five_spike_overlap": 2 }, "values": [ 5.173265526536852e-05, 5.033347770222463e-05, 4.708420601673424e-05, 4.636450466932729e-05, 6.713330367347226e-05, 6.321769615169615e-05, 6.212150037754327e-05, 6.024946924299002e-05, 7.675944652874023e-05, 7.578414806630462e-05, 7.184805872384459e-05, 6.963685154914856e-05, 8.335383608937263e-05, 8.241172326961532e-05, 7.987963181221858e-05, 7.615329377586022e-05, 8.907804294722155e-05, 8.733187860343605e-05, 8.560362039133906e-05, 8.341677312273532e-05, 9.435371612198651e-05, 9.089040395338088e-05, 8.971619536168873e-05, 8.608499047113582e-05, 9.441749716643244e-05, 9.114928980125114e-05, 8.97971258382313e-05, 8.996135875349864e-05, 9.592021524440497e-05, 9.450300422031432e-05, 9.274894546251744e-05, 8.992278890218586e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 4.8511469685763586e-05, "first_to_last_ratio": 0.5271352332510719, "imbalance_abs_log_ratio": 0.6402981537748061, "last_quartile_mean": 9.202850924339145e-05, "mean": 7.532146378252946e-05, "normalized": [ 0.548104082267993, 0.5416830199005969, 0.528884702689083, 0.5247448675895723, 0.7777813723908718, 0.7487613756360456, 0.7496059866168319, 0.7329069161583202, 0.9480676391902623, 0.9399123881532756, 0.9000936195631334, 0.883441930780847, 1.0560589316963704, 1.0534552095740957, 1.032095781768693, 0.9835620691098637, 1.1601754560133901, 1.1467788545782247, 1.1233340534265408, 1.0967375255670042, 1.2396083970871608, 1.1951467072205837, 1.1828066060767042, 1.1317734013234224, 1.244698956870482, 1.2025862582796845, 1.1881970083126314, 1.1882256015362047, 1.2689460104605979, 1.2589162611011824, 1.2307315535498626, 1.1921774555104672 ], "peak_layer": 29, "peak_normalized": 1.2689460104605979, "population_cv": 0.23486334953575583, "rest_mean": 7.254842300123225e-05, "spike_contrast": 1.244629176846487, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.029588400153444e-05, "top_five_layers": [ 29, 30, 25, 21, 31 ], "top_five_spike_overlap": 2 }, "values": [ 4.1284001781605184e-05, 4.080035796505399e-05, 3.9836369978729635e-05, 3.952455153921619e-05, 5.858363147126511e-05, 5.639780283672735e-05, 5.6461420172126964e-05, 5.520362174138427e-05, 7.140984234865755e-05, 7.079557690303773e-05, 6.77963689668104e-05, 6.654213939327747e-05, 7.954390457598493e-05, 7.934778841445222e-05, 7.773896504659206e-05, 7.408333476632833e-05, 8.738611359149218e-05, 8.637706196168438e-05, 8.461116522084922e-05, 8.26078758109361e-05, 9.336911898571998e-05, 9.002019942272455e-05, 8.909072494134307e-05, 8.524682925781235e-05, 9.375254740007222e-05, 9.058055729838088e-05, 8.949673792812973e-05, 8.949889161158353e-05, 9.557887096889317e-05, 9.482341556577012e-05, 9.270050213672221e-05, 8.979655103757977e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 9.61669284151867e-05, "first_to_last_ratio": 0.8297833624837898, "imbalance_abs_log_ratio": 0.18659062131596024, "last_quartile_mean": 0.00011589401856326731, "mean": 0.00010716335873439675, "normalized": [ 0.9380742209455786, 0.8898820858744699, 0.7837417114644826, 0.7583102664636657, 1.0822070619439073, 0.9520749081713696, 0.9107982467806989, 0.8640017807748708, 1.0669538303069324, 1.0266950649381863, 0.9594029855154438, 0.8901506143407661, 1.0795289063552718, 1.0379165674077444, 0.9706971366471856, 0.914979822300863, 1.052939971267624, 1.0225707935964177, 0.9851238030379684, 0.9636652893153618, 1.0990044510799908, 1.0523904893769196, 1.0422234788441622, 1.0049019599938585, 1.0962247910521359, 1.0599378706264646, 1.05039930496957, 1.0583562345651827, 1.1199788667078585, 1.1100529546792268, 1.0850973923636869, 1.0717171382921347 ], "peak_layer": 29, "peak_normalized": 1.1199788667078585, "population_cv": 0.09156717959150794, "rest_mean": 0.00010599351123716958, "spike_contrast": 1.0706366257223145, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00011348053521942347, "top_five_layers": [ 29, 30, 21, 25, 31 ], "top_five_spike_overlap": 2 }, "values": [ 0.00010052718425868079, 9.536275319987908e-05, 8.398839418077841e-05, 8.12630751170218e-05, 0.00011597294360399246, 0.00010202754492638633, 9.760419925441965e-05, 9.25893327803351e-05, 0.00011433835607022047, 0.00011002409155480564, 0.00010281284630764276, 9.539152961224318e-05, 0.000115685943455901, 0.0001112266254494898, 0.0001040231654769741, 9.805231093196198e-05, 0.00011283658386673778, 0.00010958212078548968, 0.00010556917550275102, 0.00010326960909878835, 0.00011777300824178383, 0.0001127776995417662, 0.00011168816854478791, 0.00010768866923172027, 0.00011747513053705916, 0.00011358650226611644, 0.00011256431753281504, 0.00011341700883349404, 0.00012002069706795737, 0.00011895700299646705, 0.00011628268111962825, 0.00011484880815260112 ] } } } } }, "parent_learned_round06_exact": true, "step": 0 }, { "local_matrix": null, "parent_learned": { "bits_per_byte": 5.150682269327325, "forward": { "activation_sha256": { "attention_branch_output": "19992b8c09cd63e9f590e656df98ebaa4fbdc484ab53c23d4c73169c5f0396ea", "mlp_branch_output": "bddd76f023c30d419ef37ac99b99ff2b543a470b0e0414eb98403c83e399da9b", "post_attention_state": "bf290f184e14eee43bbcd88e48df7e026abc6363843b7efb2c6be3c7d0a34786", "post_mlp_state": "ffee6151d0503764dd222b0cc68dfba65cc60aff853e7f321600f35337958883", "pre_attention_input": "7fe691f51fd0a4122bf92a11e80f2ae9c1ff5b2e6ee12dad9ccd027c578f75ec", "pre_mlp_input": "6c6fc562e3dd36d26a81f8fbe75bf6f7617d8035ebd514fbd7b2faa1e7aa180f" }, "logits_sha256": "bdadcb0e52fa595647848016703b410d7bcf45d1b7a624e8a3f61c3e1956509d", "loss_nats": 3.570180892944336, "mixer_summary_sha256": "023101d1bfcf41624a0444a7f14bed4c44082764bd9047e90be0fb0a456542ad" }, "loss_nats": 3.570180892944336, "loss_scale": 1.0, "mixers": [ { "branch": "attention", "entropy_mean": -0.0, "group": 1, "latest_source_mass": 1.0, "layer": 1, "max_source_mass": 1.0, "mixer_index": 0, "normalized_entropy": 1.0, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 1.0, "median": 1.0, "p05": 1.0, "p95": 1.0 } ], "sources": 1 }, { "branch": "mlp", "entropy_mean": 0.6930912733078003, "group": 1, "latest_source_mass": 0.49764734506607056, "layer": 1, "max_source_mass": 0.5023525953292847, "mixer_index": 1, "normalized_entropy": 0.9999193428845807, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.5023525953292847, "median": 0.5031890273094177, "p05": 0.4936259984970093, "p95": 0.5090169310569763 }, { "label": "current_group_partial", "mean": 0.49764734506607056, "median": 0.4968109726905823, "p05": 0.4909830391407013, "p95": 0.5063740015029907 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.692903995513916, "group": 1, "latest_source_mass": 0.48966121673583984, "layer": 2, "max_source_mass": 0.5103387832641602, "mixer_index": 2, "normalized_entropy": 0.9996491581400753, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.5103387832641602, "median": 0.5101123452186584, "p05": 0.5045775175094604, "p95": 0.5169917345046997 }, { "label": "current_group_partial", "mean": 0.48966121673583984, "median": 0.48988765478134155, "p05": 0.4830082654953003, "p95": 0.49542248249053955 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.6928629875183105, "group": 1, "latest_source_mass": 0.4946908950805664, "layer": 2, "max_source_mass": 0.5053091049194336, "mixer_index": 3, "normalized_entropy": 0.9995899961081784, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.5053091049194336, "median": 0.5084525346755981, "p05": 0.4841057062149048, "p95": 0.5181366801261902 }, { "label": "current_group_partial", "mean": 0.4946908950805664, "median": 0.49154743552207947, "p05": 0.4818633496761322, "p95": 0.5158942937850952 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6927823424339294, "group": 1, "latest_source_mass": 0.4873165488243103, "layer": 3, "max_source_mass": 0.5126834511756897, "mixer_index": 4, "normalized_entropy": 0.9994736498448697, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.5126834511756897, "median": 0.5128336548805237, "p05": 0.5050104260444641, "p95": 0.5203934907913208 }, { "label": "current_group_partial", "mean": 0.4873165488243103, "median": 0.4871663451194763, "p05": 0.4796065092086792, "p95": 0.4949895739555359 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.6928155422210693, "group": 1, "latest_source_mass": 0.49704670906066895, "layer": 3, "max_source_mass": 0.502953290939331, "mixer_index": 5, "normalized_entropy": 0.999521547013135, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.502953290939331, "median": 0.5012187957763672, "p05": 0.4864824414253235, "p95": 0.5226666927337646 }, { "label": "current_group_partial", "mean": 0.49704670906066895, "median": 0.4987812042236328, "p05": 0.4773333668708801, "p95": 0.5135174989700317 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.692751407623291, "group": 1, "latest_source_mass": 0.48708435893058777, "layer": 4, "max_source_mass": 0.5129156112670898, "mixer_index": 6, "normalized_entropy": 0.9994290203469708, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.5129156112670898, "median": 0.512997567653656, "p05": 0.504144549369812, "p95": 0.5216691493988037 }, { "label": "current_group_partial", "mean": 0.48708435893058777, "median": 0.4870024621486664, "p05": 0.4783308207988739, "p95": 0.4958554208278656 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.6925709247589111, "group": 1, "latest_source_mass": 0.48873060941696167, "layer": 4, "max_source_mass": 0.5112694501876831, "mixer_index": 7, "normalized_entropy": 0.9991686386135645, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.5112694501876831, "median": 0.5127402544021606, "p05": 0.4926953613758087, "p95": 0.5298716425895691 }, { "label": "current_group_partial", "mean": 0.48873060941696167, "median": 0.48725977540016174, "p05": 0.4701283872127533, "p95": 0.5073046088218689 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6929247379302979, "group": 2, "latest_source_mass": 0.49048691987991333, "layer": 5, "max_source_mass": 0.5095131397247314, "mixer_index": 8, "normalized_entropy": 0.9996790831213254, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.5095131397247314, "median": 0.5093363523483276, "p05": 0.502471923828125, "p95": 0.5177270770072937 }, { "label": "completed_group_1", "mean": 0.49048691987991333, "median": 0.49066367745399475, "p05": 0.4822728931903839, "p95": 0.4975281059741974 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 1.0983518362045288, "group": 2, "latest_source_mass": 0.33183109760284424, "layer": 5, "max_source_mass": 0.33983534574508667, "mixer_index": 9, "normalized_entropy": 0.999762925950977, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.33983534574508667, "median": 0.3411724865436554, "p05": 0.32688987255096436, "p95": 0.3497551679611206 }, { "label": "completed_group_1", "mean": 0.32833361625671387, "median": 0.328115850687027, "p05": 0.31867384910583496, "p95": 0.33767229318618774 }, { "label": "current_group_partial", "mean": 0.33183109760284424, "median": 0.331381618976593, "p05": 0.3268507122993469, "p95": 0.33714187145233154 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.098447322845459, "group": 2, "latest_source_mass": 0.3288182318210602, "layer": 6, "max_source_mass": 0.340813547372818, "mixer_index": 10, "normalized_entropy": 0.9998498416371705, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.340813547372818, "median": 0.340879887342453, "p05": 0.33481645584106445, "p95": 0.34668514132499695 }, { "label": "completed_group_1", "mean": 0.3303682208061218, "median": 0.33054590225219727, "p05": 0.325665682554245, "p95": 0.33449190855026245 }, { "label": "current_group_partial", "mean": 0.3288182318210602, "median": 0.3287218511104584, "p05": 0.3254890441894531, "p95": 0.33243927359580994 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0980432033538818, "group": 2, "latest_source_mass": 0.32141047716140747, "layer": 6, "max_source_mass": 0.34726741909980774, "mixer_index": 11, "normalized_entropy": 0.9994819962236925, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.34726741909980774, "median": 0.34668511152267456, "p05": 0.3404209613800049, "p95": 0.35565274953842163 }, { "label": "completed_group_1", "mean": 0.3313220739364624, "median": 0.3316044509410858, "p05": 0.3264642655849457, "p95": 0.33527836203575134 }, { "label": "current_group_partial", "mean": 0.32141047716140747, "median": 0.321965754032135, "p05": 0.3149893283843994, "p95": 0.326220840215683 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0985240936279297, "group": 2, "latest_source_mass": 0.32844245433807373, "layer": 7, "max_source_mass": 0.3373557925224304, "mixer_index": 12, "normalized_entropy": 0.9999197214148341, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.3373557925224304, "median": 0.33718767762184143, "p05": 0.33250534534454346, "p95": 0.34282490611076355 }, { "label": "completed_group_1", "mean": 0.33420175313949585, "median": 0.3341767191886902, "p05": 0.3307935893535614, "p95": 0.3377305567264557 }, { "label": "current_group_partial", "mean": 0.32844245433807373, "median": 0.3287346363067627, "p05": 0.3250963091850281, "p95": 0.33099210262298584 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.098268985748291, "group": 2, "latest_source_mass": 0.32483887672424316, "layer": 7, "max_source_mass": 0.3445510268211365, "mixer_index": 13, "normalized_entropy": 0.9996875122157655, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.3445510268211365, "median": 0.3445873260498047, "p05": 0.3388661742210388, "p95": 0.3502572774887085 }, { "label": "completed_group_1", "mean": 0.330610066652298, "median": 0.330571711063385, "p05": 0.3262380361557007, "p95": 0.3352304697036743 }, { "label": "current_group_partial", "mean": 0.32483887672424316, "median": 0.32478615641593933, "p05": 0.3213251233100891, "p95": 0.3284890055656433 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0985302925109863, "group": 2, "latest_source_mass": 0.3295760750770569, "layer": 8, "max_source_mass": 0.3371829390525818, "mixer_index": 14, "normalized_entropy": 0.9999253638813536, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.3371829390525818, "median": 0.33710986375808716, "p05": 0.330502450466156, "p95": 0.34345686435699463 }, { "label": "completed_group_1", "mean": 0.3332410156726837, "median": 0.33316272497177124, "p05": 0.32938438653945923, "p95": 0.33757662773132324 }, { "label": "current_group_partial", "mean": 0.3295760750770569, "median": 0.3295734226703644, "p05": 0.3265466094017029, "p95": 0.3326798677444458 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0983161926269531, "group": 2, "latest_source_mass": 0.32498204708099365, "layer": 8, "max_source_mass": 0.3423102796077728, "mixer_index": 15, "normalized_entropy": 0.9997304817684903, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.3423102796077728, "median": 0.34171032905578613, "p05": 0.3346346318721771, "p95": 0.35120290517807007 }, { "label": "completed_group_1", "mean": 0.3327076733112335, "median": 0.3330804705619812, "p05": 0.32750606536865234, "p95": 0.33743682503700256 }, { "label": "current_group_partial", "mean": 0.32498204708099365, "median": 0.3252435028553009, "p05": 0.31977349519729614, "p95": 0.3294060230255127 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.098489761352539, "group": 3, "latest_source_mass": 0.327616423368454, "layer": 9, "max_source_mass": 0.33793729543685913, "mixer_index": 16, "normalized_entropy": 0.9998884708310343, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.33793729543685913, "median": 0.3376197814941406, "p05": 0.3321118652820587, "p95": 0.34503501653671265 }, { "label": "completed_group_1", "mean": 0.3344462513923645, "median": 0.3347727060317993, "p05": 0.33071762323379517, "p95": 0.33719509840011597 }, { "label": "completed_group_2", "mean": 0.327616423368454, "median": 0.3275512754917145, "p05": 0.32320931553840637, "p95": 0.33203738927841187 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.3861088752746582, "group": 3, "latest_source_mass": 0.2466907799243927, "layer": 9, "max_source_mass": 0.2561652958393097, "mixer_index": 17, "normalized_entropy": 0.9998662002454641, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.2561652958393097, "median": 0.25643306970596313, "p05": 0.25058528780937195, "p95": 0.2609494626522064 }, { "label": "completed_group_1", "mean": 0.2513129711151123, "median": 0.2511752247810364, "p05": 0.24789966642856598, "p95": 0.25490516424179077 }, { "label": "completed_group_2", "mean": 0.2458309531211853, "median": 0.2456333041191101, "p05": 0.2419084906578064, "p95": 0.2499997466802597 }, { "label": "current_group_partial", "mean": 0.2466907799243927, "median": 0.246699720621109, "p05": 0.24314038455486298, "p95": 0.25063008069992065 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3861844539642334, "group": 3, "latest_source_mass": 0.24695077538490295, "layer": 10, "max_source_mass": 0.2536832094192505, "mixer_index": 18, "normalized_entropy": 0.9999207187457876, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.2536832094192505, "median": 0.253880113363266, "p05": 0.24814625084400177, "p95": 0.2593393921852112 }, { "label": "completed_group_1", "mean": 0.25214672088623047, "median": 0.2522420883178711, "p05": 0.24960598349571228, "p95": 0.2543295919895172 }, { "label": "completed_group_2", "mean": 0.24721932411193848, "median": 0.24704834818840027, "p05": 0.24384759366512299, "p95": 0.25051552057266235 }, { "label": "current_group_partial", "mean": 0.24695077538490295, "median": 0.24677348136901855, "p05": 0.24500520527362823, "p95": 0.2492748200893402 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.3860987424850464, "group": 3, "latest_source_mass": 0.24684447050094604, "layer": 10, "max_source_mass": 0.25705933570861816, "mixer_index": 19, "normalized_entropy": 0.9998588909828025, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.25705933570861816, "median": 0.2568470239639282, "p05": 0.2513251304626465, "p95": 0.26331475377082825 }, { "label": "completed_group_1", "mean": 0.24983704090118408, "median": 0.24969333410263062, "p05": 0.2465076744556427, "p95": 0.25355273485183716 }, { "label": "completed_group_2", "mean": 0.2462591528892517, "median": 0.24622508883476257, "p05": 0.24337750673294067, "p95": 0.24924828112125397 }, { "label": "current_group_partial", "mean": 0.24684447050094604, "median": 0.24683824181556702, "p05": 0.24403956532478333, "p95": 0.24949496984481812 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3862221240997314, "group": 3, "latest_source_mass": 0.24620777368545532, "layer": 11, "max_source_mass": 0.25285202264785767, "mixer_index": 20, "normalized_entropy": 0.9999478920046239, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.2514326572418213, "median": 0.25176799297332764, "p05": 0.24684108793735504, "p95": 0.2554701566696167 }, { "label": "completed_group_1", "mean": 0.25285202264785767, "median": 0.25268346071243286, "p05": 0.2510001063346863, "p95": 0.2550324499607086 }, { "label": "completed_group_2", "mean": 0.24950754642486572, "median": 0.2494041621685028, "p05": 0.2475442737340927, "p95": 0.2515066862106323 }, { "label": "current_group_partial", "mean": 0.24620777368545532, "median": 0.24642038345336914, "p05": 0.2445429414510727, "p95": 0.2473805695772171 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.386016845703125, "group": 3, "latest_source_mass": 0.24290117621421814, "layer": 11, "max_source_mass": 0.25482746958732605, "mixer_index": 21, "normalized_entropy": 0.999799814942231, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.25482746958732605, "median": 0.2545710802078247, "p05": 0.2466055005788803, "p95": 0.2635689973831177 }, { "label": "completed_group_1", "mean": 0.2536851167678833, "median": 0.2534823715686798, "p05": 0.2508774995803833, "p95": 0.2568957209587097 }, { "label": "completed_group_2", "mean": 0.24858617782592773, "median": 0.24861112236976624, "p05": 0.24390698969364166, "p95": 0.252436101436615 }, { "label": "current_group_partial", "mean": 0.24290117621421814, "median": 0.24232372641563416, "p05": 0.23918122053146362, "p95": 0.24772994220256805 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3862277269363403, "group": 3, "latest_source_mass": 0.24656552076339722, "layer": 12, "max_source_mass": 0.2530350983142853, "mixer_index": 22, "normalized_entropy": 0.9999519335969191, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.25171470642089844, "median": 0.2513781189918518, "p05": 0.24833296239376068, "p95": 0.25546184182167053 }, { "label": "completed_group_1", "mean": 0.2530350983142853, "median": 0.25303736329078674, "p05": 0.25148919224739075, "p95": 0.2544969618320465 }, { "label": "completed_group_2", "mean": 0.24868467450141907, "median": 0.24892255663871765, "p05": 0.2467103898525238, "p95": 0.25010940432548523 }, { "label": "current_group_partial", "mean": 0.24656552076339722, "median": 0.24677836894989014, "p05": 0.24485518038272858, "p95": 0.2477675825357437 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.3859888315200806, "group": 3, "latest_source_mass": 0.24197682738304138, "layer": 12, "max_source_mass": 0.2546530067920685, "mixer_index": 23, "normalized_entropy": 0.9997796069807546, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.25458407402038574, "median": 0.2539653182029724, "p05": 0.24643409252166748, "p95": 0.26325899362564087 }, { "label": "completed_group_1", "mean": 0.2546530067920685, "median": 0.2548711895942688, "p05": 0.25055474042892456, "p95": 0.25881263613700867 }, { "label": "completed_group_2", "mean": 0.248786062002182, "median": 0.24910664558410645, "p05": 0.2454056739807129, "p95": 0.2517758011817932 }, { "label": "current_group_partial", "mean": 0.24197682738304138, "median": 0.2416902482509613, "p05": 0.23907747864723206, "p95": 0.2456972599029541 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3862541913986206, "group": 4, "latest_source_mass": 0.24767187237739563, "layer": 13, "max_source_mass": 0.2521768808364868, "mixer_index": 24, "normalized_entropy": 0.999971023671165, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.2501963973045349, "median": 0.250087708234787, "p05": 0.24644337594509125, "p95": 0.25403982400894165 }, { "label": "completed_group_1", "mean": 0.2521768808364868, "median": 0.2521744668483734, "p05": 0.25072887539863586, "p95": 0.25345146656036377 }, { "label": "completed_group_2", "mean": 0.24995486438274384, "median": 0.2498224377632141, "p05": 0.2482302337884903, "p95": 0.2518054246902466 }, { "label": "completed_group_3", "mean": 0.24767187237739563, "median": 0.2475992739200592, "p05": 0.24603278934955597, "p95": 0.2497682422399521 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.6090729236602783, "group": 4, "latest_source_mass": 0.19462624192237854, "layer": 13, "max_source_mass": 0.20631682872772217, "mixer_index": 25, "normalized_entropy": 0.9997732197241024, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.20360878109931946, "median": 0.20399323105812073, "p05": 0.19613273441791534, "p95": 0.21039927005767822 }, { "label": "completed_group_1", "mean": 0.20631682872772217, "median": 0.20582494139671326, "p05": 0.20368342101573944, "p95": 0.2099049985408783 }, { "label": "completed_group_2", "mean": 0.19986985623836517, "median": 0.19948531687259674, "p05": 0.19628161191940308, "p95": 0.20406028628349304 }, { "label": "completed_group_3", "mean": 0.19557830691337585, "median": 0.19561558961868286, "p05": 0.1910029649734497, "p95": 0.20039188861846924 }, { "label": "current_group_partial", "mean": 0.19462624192237854, "median": 0.1954103410243988, "p05": 0.19068367779254913, "p95": 0.19781020283699036 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.6093083620071411, "group": 4, "latest_source_mass": 0.1976698487997055, "layer": 14, "max_source_mass": 0.20362424850463867, "mixer_index": 26, "normalized_entropy": 0.9999195057939432, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.2017374038696289, "median": 0.2026747167110443, "p05": 0.19666710495948792, "p95": 0.20696133375167847 }, { "label": "completed_group_1", "mean": 0.20362424850463867, "median": 0.20354315638542175, "p05": 0.20209211111068726, "p95": 0.20522044599056244 }, { "label": "completed_group_2", "mean": 0.20011675357818604, "median": 0.1997961699962616, "p05": 0.19781088829040527, "p95": 0.2023434340953827 }, { "label": "completed_group_3", "mean": 0.1968517303466797, "median": 0.19661611318588257, "p05": 0.19452345371246338, "p95": 0.19933368265628815 }, { "label": "current_group_partial", "mean": 0.1976698487997055, "median": 0.19780653715133667, "p05": 0.19629782438278198, "p95": 0.19887655973434448 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.608644723892212, "group": 4, "latest_source_mass": 0.18768835067749023, "layer": 14, "max_source_mass": 0.20831988751888275, "mixer_index": 27, "normalized_entropy": 0.9995071642492324, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.20431089401245117, "median": 0.20455676317214966, "p05": 0.19753149151802063, "p95": 0.21038773655891418 }, { "label": "completed_group_1", "mean": 0.20831988751888275, "median": 0.20877230167388916, "p05": 0.2030324637889862, "p95": 0.2128065973520279 }, { "label": "completed_group_2", "mean": 0.20422273874282837, "median": 0.2047901451587677, "p05": 0.20030567049980164, "p95": 0.20715001225471497 }, { "label": "completed_group_3", "mean": 0.19545812904834747, "median": 0.19552549719810486, "p05": 0.1923498660326004, "p95": 0.19825047254562378 }, { "label": "current_group_partial", "mean": 0.18768835067749023, "median": 0.18748146295547485, "p05": 0.18511706590652466, "p95": 0.19059684872627258 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.609345555305481, "group": 4, "latest_source_mass": 0.19806164503097534, "layer": 15, "max_source_mass": 0.20386312901973724, "mixer_index": 28, "normalized_entropy": 0.9999426152895332, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.2002599537372589, "median": 0.20108085870742798, "p05": 0.19542044401168823, "p95": 0.20405235886573792 }, { "label": "completed_group_1", "mean": 0.20386312901973724, "median": 0.20386025309562683, "p05": 0.20271247625350952, "p95": 0.20498213171958923 }, { "label": "completed_group_2", "mean": 0.2001732587814331, "median": 0.19999440014362335, "p05": 0.1985137164592743, "p95": 0.2019573152065277 }, { "label": "completed_group_3", "mean": 0.1976420283317566, "median": 0.19760513305664062, "p05": 0.1957571804523468, "p95": 0.20000392198562622 }, { "label": "current_group_partial", "mean": 0.19806164503097534, "median": 0.19796663522720337, "p05": 0.1969158947467804, "p95": 0.19929495453834534 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.6086196899414062, "group": 4, "latest_source_mass": 0.18893863260746002, "layer": 15, "max_source_mass": 0.21025606989860535, "mixer_index": 29, "normalized_entropy": 0.9994916097810468, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.20515945553779602, "median": 0.2049580067396164, "p05": 0.19868221879005432, "p95": 0.21147878468036652 }, { "label": "completed_group_1", "mean": 0.21025606989860535, "median": 0.2101147472858429, "p05": 0.20689907670021057, "p95": 0.21399712562561035 }, { "label": "completed_group_2", "mean": 0.20179016888141632, "median": 0.20176216959953308, "p05": 0.199407696723938, "p95": 0.20402206480503082 }, { "label": "completed_group_3", "mean": 0.1938556730747223, "median": 0.19379490613937378, "p05": 0.19004228711128235, "p95": 0.1977127194404602 }, { "label": "current_group_partial", "mean": 0.18893863260746002, "median": 0.1887960582971573, "p05": 0.18639500439167023, "p95": 0.19145210087299347 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.609370470046997, "group": 4, "latest_source_mass": 0.1981920301914215, "layer": 16, "max_source_mass": 0.20340535044670105, "mixer_index": 30, "normalized_entropy": 0.9999580956888227, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.2002836912870407, "median": 0.20032534003257751, "p05": 0.19668522477149963, "p95": 0.20378869771957397 }, { "label": "completed_group_1", "mean": 0.20340535044670105, "median": 0.20332223176956177, "p05": 0.2024117112159729, "p95": 0.20457619428634644 }, { "label": "completed_group_2", "mean": 0.20010054111480713, "median": 0.1998724788427353, "p05": 0.19848622381687164, "p95": 0.20170091092586517 }, { "label": "completed_group_3", "mean": 0.1980184018611908, "median": 0.19809064269065857, "p05": 0.19669371843338013, "p95": 0.19908645749092102 }, { "label": "current_group_partial", "mean": 0.1981920301914215, "median": 0.19810855388641357, "p05": 0.1971922218799591, "p95": 0.19958281517028809 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.6083989143371582, "group": 4, "latest_source_mass": 0.18735264241695404, "layer": 16, "max_source_mass": 0.21095217764377594, "mixer_index": 31, "normalized_entropy": 0.9993544341854289, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.20363345742225647, "median": 0.20384372770786285, "p05": 0.19405215978622437, "p95": 0.21326100826263428 }, { "label": "completed_group_1", "mean": 0.21095217764377594, "median": 0.21146853268146515, "p05": 0.20567592978477478, "p95": 0.21522513031959534 }, { "label": "completed_group_2", "mean": 0.20392385125160217, "median": 0.20381325483322144, "p05": 0.20115600526332855, "p95": 0.2067122757434845 }, { "label": "completed_group_3", "mean": 0.19413790106773376, "median": 0.19391843676567078, "p05": 0.18873284757137299, "p95": 0.1996975690126419 }, { "label": "current_group_partial", "mean": 0.18735264241695404, "median": 0.18673613667488098, "p05": 0.1829182654619217, "p95": 0.1927962601184845 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.6093392372131348, "group": 5, "latest_source_mass": 0.19563516974449158, "layer": 17, "max_source_mass": 0.20331618189811707, "mixer_index": 32, "normalized_entropy": 0.9999386896380388, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.20084109902381897, "median": 0.20074692368507385, "p05": 0.198189377784729, "p95": 0.20358796417713165 }, { "label": "completed_group_1", "mean": 0.20331618189811707, "median": 0.20337940752506256, "p05": 0.20178094506263733, "p95": 0.20489434897899628 }, { "label": "completed_group_2", "mean": 0.20119993388652802, "median": 0.20122915506362915, "p05": 0.20026232302188873, "p95": 0.2019881308078766 }, { "label": "completed_group_3", "mean": 0.19900763034820557, "median": 0.198960542678833, "p05": 0.19746547937393188, "p95": 0.20062875747680664 }, { "label": "completed_group_4", "mean": 0.19563516974449158, "median": 0.19558461010456085, "p05": 0.193980410695076, "p95": 0.19730384647846222 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.7908520698547363, "group": 5, "latest_source_mass": 0.1625179648399353, "layer": 17, "max_source_mass": 0.17730244994163513, "mixer_index": 33, "normalized_entropy": 0.9994935707672249, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.17160072922706604, "median": 0.1718936264514923, "p05": 0.16487310826778412, "p95": 0.17849747836589813 }, { "label": "completed_group_1", "mean": 0.17730244994163513, "median": 0.1778835654258728, "p05": 0.17289136350154877, "p95": 0.18126235902309418 }, { "label": "completed_group_2", "mean": 0.1690303534269333, "median": 0.16928547620773315, "p05": 0.16646793484687805, "p95": 0.17125293612480164 }, { "label": "completed_group_3", "mean": 0.16099892556667328, "median": 0.16061408817768097, "p05": 0.15715192258358002, "p95": 0.1656636893749237 }, { "label": "completed_group_4", "mean": 0.15854957699775696, "median": 0.15825147926807404, "p05": 0.15517033636569977, "p95": 0.1620505452156067 }, { "label": "current_group_partial", "mean": 0.1625179648399353, "median": 0.16315963864326477, "p05": 0.15939058363437653, "p95": 0.16489268839359283 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.7916775941848755, "group": 5, "latest_source_mass": 0.16463731229305267, "layer": 18, "max_source_mass": 0.16981256008148193, "mixer_index": 34, "normalized_entropy": 0.9999543046683522, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.16706335544586182, "median": 0.16679415106773376, "p05": 0.16531850397586823, "p95": 0.16906341910362244 }, { "label": "completed_group_1", "mean": 0.16981256008148193, "median": 0.1699288785457611, "p05": 0.1682884395122528, "p95": 0.17106440663337708 }, { "label": "completed_group_2", "mean": 0.16858771443367004, "median": 0.16860879957675934, "p05": 0.16782470047473907, "p95": 0.16934865713119507 }, { "label": "completed_group_3", "mean": 0.16527186334133148, "median": 0.16520482301712036, "p05": 0.1643052101135254, "p95": 0.16638660430908203 }, { "label": "completed_group_4", "mean": 0.16462719440460205, "median": 0.16459274291992188, "p05": 0.1638425886631012, "p95": 0.16539013385772705 }, { "label": "current_group_partial", "mean": 0.16463731229305267, "median": 0.16451048851013184, "p05": 0.1639435589313507, "p95": 0.16552197933197021 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.7898297309875488, "group": 5, "latest_source_mass": 0.153368279337883, "layer": 18, "max_source_mass": 0.18101517856121063, "mixer_index": 35, "normalized_entropy": 0.9989229925815112, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.17284835875034332, "median": 0.17269954085350037, "p05": 0.1658562421798706, "p95": 0.18014436960220337 }, { "label": "completed_group_1", "mean": 0.18101517856121063, "median": 0.1814284771680832, "p05": 0.17416825890541077, "p95": 0.18707150220870972 }, { "label": "completed_group_2", "mean": 0.1735592633485794, "median": 0.17370939254760742, "p05": 0.17075185477733612, "p95": 0.17625606060028076 }, { "label": "completed_group_3", "mean": 0.16231875121593475, "median": 0.1621963381767273, "p05": 0.15821577608585358, "p95": 0.16650474071502686 }, { "label": "completed_group_4", "mean": 0.15689018368721008, "median": 0.15674838423728943, "p05": 0.15215492248535156, "p95": 0.1616627275943756 }, { "label": "current_group_partial", "mean": 0.153368279337883, "median": 0.1528519243001938, "p05": 0.15056025981903076, "p95": 0.15731072425842285 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.7917213439941406, "group": 5, "latest_source_mass": 0.16463561356067657, "layer": 19, "max_source_mass": 0.16885727643966675, "mixer_index": 36, "normalized_entropy": 0.9999787219018127, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1661592274904251, "median": 0.1661478579044342, "p05": 0.16439414024353027, "p95": 0.1678619235754013 }, { "label": "completed_group_1", "mean": 0.16885727643966675, "median": 0.16877895593643188, "p05": 0.16819745302200317, "p95": 0.1697411835193634 }, { "label": "completed_group_2", "mean": 0.16764025390148163, "median": 0.16765299439430237, "p05": 0.1669837087392807, "p95": 0.1682416945695877 }, { "label": "completed_group_3", "mean": 0.16661542654037476, "median": 0.1665019690990448, "p05": 0.1658560335636139, "p95": 0.16755254566669464 }, { "label": "completed_group_4", "mean": 0.16609220206737518, "median": 0.16612619161605835, "p05": 0.16555683314800262, "p95": 0.16653065383434296 }, { "label": "current_group_partial", "mean": 0.16463561356067657, "median": 0.16466060280799866, "p05": 0.16384081542491913, "p95": 0.16551899909973145 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.7890949249267578, "group": 5, "latest_source_mass": 0.15094393491744995, "layer": 19, "max_source_mass": 0.18413972854614258, "mixer_index": 37, "normalized_entropy": 0.9985128895105295, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.17241930961608887, "median": 0.17162901163101196, "p05": 0.1626318097114563, "p95": 0.18299198150634766 }, { "label": "completed_group_1", "mean": 0.18413972854614258, "median": 0.1848709136247635, "p05": 0.17655718326568604, "p95": 0.19053134322166443 }, { "label": "completed_group_2", "mean": 0.17484498023986816, "median": 0.17485198378562927, "p05": 0.1718076914548874, "p95": 0.17815767228603363 }, { "label": "completed_group_3", "mean": 0.1621202528476715, "median": 0.16197144985198975, "p05": 0.15698590874671936, "p95": 0.16731801629066467 }, { "label": "completed_group_4", "mean": 0.15553176403045654, "median": 0.15512266755104065, "p05": 0.1502593755722046, "p95": 0.16059254109859467 }, { "label": "current_group_partial", "mean": 0.15094393491744995, "median": 0.15083211660385132, "p05": 0.1470862179994583, "p95": 0.1549530029296875 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.7917208671569824, "group": 5, "latest_source_mass": 0.16669265925884247, "layer": 20, "max_source_mass": 0.168935626745224, "mixer_index": 38, "normalized_entropy": 0.9999784557739275, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.16663745045661926, "median": 0.16669239103794098, "p05": 0.16406413912773132, "p95": 0.16939817368984222 }, { "label": "completed_group_1", "mean": 0.168935626745224, "median": 0.16887718439102173, "p05": 0.16814815998077393, "p95": 0.1698606014251709 }, { "label": "completed_group_2", "mean": 0.16597259044647217, "median": 0.16582076251506805, "p05": 0.16482511162757874, "p95": 0.1670236438512802 }, { "label": "completed_group_3", "mean": 0.16514870524406433, "median": 0.165022611618042, "p05": 0.16397784650325775, "p95": 0.1662973016500473 }, { "label": "completed_group_4", "mean": 0.16661295294761658, "median": 0.16667577624320984, "p05": 0.16593852639198303, "p95": 0.16715332865715027 }, { "label": "current_group_partial", "mean": 0.16669265925884247, "median": 0.1666145920753479, "p05": 0.16592761874198914, "p95": 0.1674128621816635 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.789101004600525, "group": 5, "latest_source_mass": 0.14993149042129517, "layer": 20, "max_source_mass": 0.18365857005119324, "mixer_index": 39, "normalized_entropy": 0.9985162826410648, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.17229264974594116, "median": 0.1720556616783142, "p05": 0.16247707605361938, "p95": 0.18216519057750702 }, { "label": "completed_group_1", "mean": 0.18365857005119324, "median": 0.18395906686782837, "p05": 0.17734748125076294, "p95": 0.18972161412239075 }, { "label": "completed_group_2", "mean": 0.1747075319290161, "median": 0.1746184229850769, "p05": 0.17210862040519714, "p95": 0.1774953156709671 }, { "label": "completed_group_3", "mean": 0.1640234887599945, "median": 0.1635349988937378, "p05": 0.15958356857299805, "p95": 0.16904467344284058 }, { "label": "completed_group_4", "mean": 0.15538626909255981, "median": 0.15498605370521545, "p05": 0.14995241165161133, "p95": 0.16101743280887604 }, { "label": "current_group_partial", "mean": 0.14993149042129517, "median": 0.15014703571796417, "p05": 0.14564283192157745, "p95": 0.1527552753686905 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.7917447090148926, "group": 6, "latest_source_mass": 0.1661863625049591, "layer": 21, "max_source_mass": 0.16768339276313782, "mixer_index": 40, "normalized_entropy": 0.9999917621681839, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1658121943473816, "median": 0.1657177209854126, "p05": 0.16437731683254242, "p95": 0.16778677701950073 }, { "label": "completed_group_1", "mean": 0.16768339276313782, "median": 0.16760629415512085, "p05": 0.1666821986436844, "p95": 0.16875657439231873 }, { "label": "completed_group_2", "mean": 0.16629494726657867, "median": 0.16622786223888397, "p05": 0.1655496209859848, "p95": 0.16718709468841553 }, { "label": "completed_group_3", "mean": 0.1669958382844925, "median": 0.16695073246955872, "p05": 0.1664678305387497, "p95": 0.16760030388832092 }, { "label": "completed_group_4", "mean": 0.16702723503112793, "median": 0.16713355481624603, "p05": 0.16618871688842773, "p95": 0.16781742870807648 }, { "label": "completed_group_5", "mean": 0.1661863625049591, "median": 0.16608929634094238, "p05": 0.16534176468849182, "p95": 0.16705378890037537 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.9436898231506348, "group": 6, "latest_source_mass": 0.13651785254478455, "layer": 21, "max_source_mass": 0.15856821835041046, "mixer_index": 41, "normalized_entropy": 0.9988589781980651, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.14778707921504974, "median": 0.14752492308616638, "p05": 0.13982102274894714, "p95": 0.1569063663482666 }, { "label": "completed_group_1", "mean": 0.15856821835041046, "median": 0.1585446149110794, "p05": 0.15396656095981598, "p95": 0.16390106081962585 }, { "label": "completed_group_2", "mean": 0.1500256508588791, "median": 0.14993107318878174, "p05": 0.14748509228229523, "p95": 0.15272916853427887 }, { "label": "completed_group_3", "mean": 0.1408001333475113, "median": 0.14052921533584595, "p05": 0.13647492229938507, "p95": 0.14521925151348114 }, { "label": "completed_group_4", "mean": 0.13533657789230347, "median": 0.13547882437705994, "p05": 0.13068656623363495, "p95": 0.13946673274040222 }, { "label": "completed_group_5", "mean": 0.130964457988739, "median": 0.13209164142608643, "p05": 0.12633010745048523, "p95": 0.13486123085021973 }, { "label": "current_group_partial", "mean": 0.13651785254478455, "median": 0.1365797072649002, "p05": 0.13271403312683105, "p95": 0.14000433683395386 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9458942413330078, "group": 6, "latest_source_mass": 0.14246684312820435, "layer": 22, "max_source_mass": 0.14393000304698944, "mixer_index": 42, "normalized_entropy": 0.9999918250478763, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.142832413315773, "median": 0.14290162920951843, "p05": 0.14148348569869995, "p95": 0.1440550833940506 }, { "label": "completed_group_1", "mean": 0.1429906189441681, "median": 0.14299973845481873, "p05": 0.14231431484222412, "p95": 0.14357265830039978 }, { "label": "completed_group_2", "mean": 0.14157448709011078, "median": 0.14162302017211914, "p05": 0.1408008486032486, "p95": 0.1421196460723877 }, { "label": "completed_group_3", "mean": 0.14285019040107727, "median": 0.14287817478179932, "p05": 0.1423148512840271, "p95": 0.14345195889472961 }, { "label": "completed_group_4", "mean": 0.14393000304698944, "median": 0.14395871758460999, "p05": 0.14325706660747528, "p95": 0.14449745416641235 }, { "label": "completed_group_5", "mean": 0.14335544407367706, "median": 0.14326418936252594, "p05": 0.14292818307876587, "p95": 0.14392802119255066 }, { "label": "current_group_partial", "mean": 0.14246684312820435, "median": 0.14247724413871765, "p05": 0.14211606979370117, "p95": 0.14281725883483887 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.9425597190856934, "group": 6, "latest_source_mass": 0.12947803735733032, "layer": 22, "max_source_mass": 0.1615946888923645, "mixer_index": 43, "normalized_entropy": 0.9982782195923864, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1483290195465088, "median": 0.14828777313232422, "p05": 0.13995377719402313, "p95": 0.15759113430976868 }, { "label": "completed_group_1", "mean": 0.1615946888923645, "median": 0.16178560256958008, "p05": 0.15419340133666992, "p95": 0.16922280192375183 }, { "label": "completed_group_2", "mean": 0.15280696749687195, "median": 0.1528012454509735, "p05": 0.15010300278663635, "p95": 0.1558753252029419 }, { "label": "completed_group_3", "mean": 0.14239415526390076, "median": 0.14164817333221436, "p05": 0.1380041539669037, "p95": 0.14742238819599152 }, { "label": "completed_group_4", "mean": 0.13462956249713898, "median": 0.1342291682958603, "p05": 0.12952053546905518, "p95": 0.14025796949863434 }, { "label": "completed_group_5", "mean": 0.1307675838470459, "median": 0.13075318932533264, "p05": 0.12602169811725616, "p95": 0.13501127064228058 }, { "label": "current_group_partial", "mean": 0.12947803735733032, "median": 0.1295393407344818, "p05": 0.1264311969280243, "p95": 0.13163118064403534 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9458987712860107, "group": 6, "latest_source_mass": 0.14346569776535034, "layer": 23, "max_source_mass": 0.14346569776535034, "mixer_index": 44, "normalized_entropy": 0.9999941529832156, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1426449418067932, "median": 0.14265665411949158, "p05": 0.14160317182540894, "p95": 0.1436314880847931 }, { "label": "completed_group_1", "mean": 0.14291533827781677, "median": 0.14293473958969116, "p05": 0.1422625482082367, "p95": 0.14348529279232025 }, { "label": "completed_group_2", "mean": 0.1416938751935959, "median": 0.14168253540992737, "p05": 0.14088502526283264, "p95": 0.14245855808258057 }, { "label": "completed_group_3", "mean": 0.14274153113365173, "median": 0.14274577796459198, "p05": 0.14230182766914368, "p95": 0.14324283599853516 }, { "label": "completed_group_4", "mean": 0.14325544238090515, "median": 0.1432678997516632, "p05": 0.14277629554271698, "p95": 0.14376230537891388 }, { "label": "completed_group_5", "mean": 0.1432831883430481, "median": 0.14325812458992004, "p05": 0.14287351071834564, "p95": 0.1437322348356247 }, { "label": "current_group_partial", "mean": 0.14346569776535034, "median": 0.1435171663761139, "p05": 0.142959862947464, "p95": 0.1437678039073944 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.941068410873413, "group": 6, "latest_source_mass": 0.1254161298274994, "layer": 23, "max_source_mass": 0.16456490755081177, "mixer_index": 45, "normalized_entropy": 0.9975118387741332, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.14912807941436768, "median": 0.14865416288375854, "p05": 0.13610364496707916, "p95": 0.16207551956176758 }, { "label": "completed_group_1", "mean": 0.16456490755081177, "median": 0.16586321592330933, "p05": 0.15482422709465027, "p95": 0.17167139053344727 }, { "label": "completed_group_2", "mean": 0.15460123121738434, "median": 0.1543833166360855, "p05": 0.15155401825904846, "p95": 0.15836837887763977 }, { "label": "completed_group_3", "mean": 0.1425517499446869, "median": 0.14188045263290405, "p05": 0.13683314621448517, "p95": 0.1494884341955185 }, { "label": "completed_group_4", "mean": 0.13539987802505493, "median": 0.13422535359859467, "p05": 0.12956365942955017, "p95": 0.14296601712703705 }, { "label": "completed_group_5", "mean": 0.1283380389213562, "median": 0.12796106934547424, "p05": 0.12316946685314178, "p95": 0.13433398306369781 }, { "label": "current_group_partial", "mean": 0.1254161298274994, "median": 0.12523823976516724, "p05": 0.12156305462121964, "p95": 0.12899920344352722 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9458861351013184, "group": 6, "latest_source_mass": 0.14452049136161804, "layer": 24, "max_source_mass": 0.14452049136161804, "mixer_index": 46, "normalized_entropy": 0.9999876592688483, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.14264336228370667, "median": 0.1426892876625061, "p05": 0.14129234850406647, "p95": 0.14382319152355194 }, { "label": "completed_group_1", "mean": 0.1430070698261261, "median": 0.14298146963119507, "p05": 0.14245536923408508, "p95": 0.1435846984386444 }, { "label": "completed_group_2", "mean": 0.14314037561416626, "median": 0.14310884475708008, "p05": 0.14277565479278564, "p95": 0.14360389113426208 }, { "label": "completed_group_3", "mean": 0.14221979677677155, "median": 0.14216911792755127, "p05": 0.14183637499809265, "p95": 0.14285607635974884 }, { "label": "completed_group_4", "mean": 0.14132915437221527, "median": 0.1413242518901825, "p05": 0.140913724899292, "p95": 0.14174436032772064 }, { "label": "completed_group_5", "mean": 0.1431397795677185, "median": 0.14310240745544434, "p05": 0.14276552200317383, "p95": 0.14377698302268982 }, { "label": "current_group_partial", "mean": 0.14452049136161804, "median": 0.1444905698299408, "p05": 0.14409083127975464, "p95": 0.14507406949996948 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.9415628910064697, "group": 6, "latest_source_mass": 0.1248510330915451, "layer": 24, "max_source_mass": 0.16342055797576904, "mixer_index": 47, "normalized_entropy": 0.9977659512948458, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.14895881712436676, "median": 0.14996516704559326, "p05": 0.13763123750686646, "p95": 0.15872029960155487 }, { "label": "completed_group_1", "mean": 0.16342055797576904, "median": 0.1639186143875122, "p05": 0.15447522699832916, "p95": 0.17102839052677155 }, { "label": "completed_group_2", "mean": 0.15406624972820282, "median": 0.1540084183216095, "p05": 0.1514483094215393, "p95": 0.1571783721446991 }, { "label": "completed_group_3", "mean": 0.14293310046195984, "median": 0.14186778664588928, "p05": 0.13832569122314453, "p95": 0.1491597443819046 }, { "label": "completed_group_4", "mean": 0.1363803744316101, "median": 0.13557225465774536, "p05": 0.13139937818050385, "p95": 0.14291447401046753 }, { "label": "completed_group_5", "mean": 0.12938985228538513, "median": 0.12914851307868958, "p05": 0.12471500039100647, "p95": 0.13491052389144897 }, { "label": "current_group_partial", "mean": 0.1248510330915451, "median": 0.12450477480888367, "p05": 0.120968759059906, "p95": 0.12821169197559357 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9458723068237305, "group": 7, "latest_source_mass": 0.14437910914421082, "layer": 25, "max_source_mass": 0.14437910914421082, "mixer_index": 48, "normalized_entropy": 0.999980552939918, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.14191991090774536, "median": 0.14168427884578705, "p05": 0.14056351780891418, "p95": 0.14397425949573517 }, { "label": "completed_group_1", "mean": 0.1418565809726715, "median": 0.1417136788368225, "p05": 0.14059004187583923, "p95": 0.14328429102897644 }, { "label": "completed_group_2", "mean": 0.14164230227470398, "median": 0.14142388105392456, "p05": 0.14061963558197021, "p95": 0.14320334792137146 }, { "label": "completed_group_3", "mean": 0.14289972186088562, "median": 0.14285793900489807, "p05": 0.1423202008008957, "p95": 0.14347271621227264 }, { "label": "completed_group_4", "mean": 0.14397987723350525, "median": 0.1439221203327179, "p05": 0.14329849183559418, "p95": 0.14462903141975403 }, { "label": "completed_group_5", "mean": 0.14332249760627747, "median": 0.14335353672504425, "p05": 0.14246267080307007, "p95": 0.14414797723293304 }, { "label": "completed_group_6", "mean": 0.14437910914421082, "median": 0.14459103345870972, "p05": 0.14338219165802002, "p95": 0.14524540305137634 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 2.073223114013672, "group": 7, "latest_source_mass": 0.11086538434028625, "layer": 25, "max_source_mass": 0.14827430248260498, "mixer_index": 49, "normalized_entropy": 0.9970095684146328, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.13280925154685974, "median": 0.13328123092651367, "p05": 0.11957645416259766, "p95": 0.1447097212076187 }, { "label": "completed_group_1", "mean": 0.14827430248260498, "median": 0.14952056109905243, "p05": 0.13636183738708496, "p95": 0.15699218213558197 }, { "label": "completed_group_2", "mean": 0.13796664774417877, "median": 0.1381954699754715, "p05": 0.13450786471366882, "p95": 0.1413417011499405 }, { "label": "completed_group_3", "mean": 0.12756124138832092, "median": 0.12679177522659302, "p05": 0.12232367694377899, "p95": 0.13387270271778107 }, { "label": "completed_group_4", "mean": 0.12019774317741394, "median": 0.11888951063156128, "p05": 0.11402077227830887, "p95": 0.12811852991580963 }, { "label": "completed_group_5", "mean": 0.11310221254825592, "median": 0.11254298686981201, "p05": 0.10767743736505508, "p95": 0.12006338685750961 }, { "label": "completed_group_6", "mean": 0.10922321677207947, "median": 0.10910944640636444, "p05": 0.10472994297742844, "p95": 0.11491495370864868 }, { "label": "current_group_partial", "mean": 0.11086538434028625, "median": 0.11154524236917496, "p05": 0.10694204270839691, "p95": 0.11396989226341248 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.079413890838623, "group": 7, "latest_source_mass": 0.1263430416584015, "layer": 26, "max_source_mass": 0.1263430416584015, "mixer_index": 50, "normalized_entropy": 0.9999867027561686, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.12475188821554184, "median": 0.12463723123073578, "p05": 0.12364111095666885, "p95": 0.1259622424840927 }, { "label": "completed_group_1", "mean": 0.12460128962993622, "median": 0.12450964748859406, "p05": 0.12383783608675003, "p95": 0.125508114695549 }, { "label": "completed_group_2", "mean": 0.12424702197313309, "median": 0.12409250438213348, "p05": 0.1235705241560936, "p95": 0.12524327635765076 }, { "label": "completed_group_3", "mean": 0.12420950829982758, "median": 0.12421596050262451, "p05": 0.12374235689640045, "p95": 0.12457412481307983 }, { "label": "completed_group_4", "mean": 0.12484367191791534, "median": 0.12490126490592957, "p05": 0.12441762536764145, "p95": 0.1252550482749939 }, { "label": "completed_group_5", "mean": 0.12469653785228729, "median": 0.12477756291627884, "p05": 0.12382431328296661, "p95": 0.12524525821208954 }, { "label": "completed_group_6", "mean": 0.12630701065063477, "median": 0.12644168734550476, "p05": 0.12546542286872864, "p95": 0.12699897587299347 }, { "label": "current_group_partial", "mean": 0.1263430416584015, "median": 0.12645362317562103, "p05": 0.12567421793937683, "p95": 0.1267925351858139 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.072925567626953, "group": 7, "latest_source_mass": 0.10599855333566666, "layer": 26, "max_source_mass": 0.14843900501728058, "mixer_index": 51, "normalized_entropy": 0.9968664788491151, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.13228744268417358, "median": 0.13281425833702087, "p05": 0.12057555466890335, "p95": 0.14273115992546082 }, { "label": "completed_group_1", "mean": 0.14843900501728058, "median": 0.14951951801776886, "p05": 0.13698264956474304, "p95": 0.1572267711162567 }, { "label": "completed_group_2", "mean": 0.13848665356636047, "median": 0.1387266218662262, "p05": 0.13503478467464447, "p95": 0.1420348733663559 }, { "label": "completed_group_3", "mean": 0.12843601405620575, "median": 0.12770286202430725, "p05": 0.12394566833972931, "p95": 0.13404949009418488 }, { "label": "completed_group_4", "mean": 0.12169650197029114, "median": 0.12072620540857315, "p05": 0.11642728000879288, "p95": 0.12845680117607117 }, { "label": "completed_group_5", "mean": 0.11442162096500397, "median": 0.11409753561019897, "p05": 0.10971812158823013, "p95": 0.12062239646911621 }, { "label": "completed_group_6", "mean": 0.11023421585559845, "median": 0.11024647206068039, "p05": 0.10620385408401489, "p95": 0.11548472940921783 }, { "label": "current_group_partial", "mean": 0.10599855333566666, "median": 0.10623474419116974, "p05": 0.1023266613483429, "p95": 0.10840564221143723 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.0793263912200928, "group": 7, "latest_source_mass": 0.12782885134220123, "layer": 27, "max_source_mass": 0.12782885134220123, "mixer_index": 52, "normalized_entropy": 0.9999446243342576, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.12420415878295898, "median": 0.12419730424880981, "p05": 0.12226121127605438, "p95": 0.12590435147285461 }, { "label": "completed_group_1", "mean": 0.12273682653903961, "median": 0.12260904908180237, "p05": 0.12171939015388489, "p95": 0.12380002439022064 }, { "label": "completed_group_2", "mean": 0.12360995262861252, "median": 0.12356993556022644, "p05": 0.12293750047683716, "p95": 0.12423694133758545 }, { "label": "completed_group_3", "mean": 0.1238352432847023, "median": 0.12381016463041306, "p05": 0.12339816987514496, "p95": 0.12445184588432312 }, { "label": "completed_group_4", "mean": 0.1241140365600586, "median": 0.12404628843069077, "p05": 0.12358136475086212, "p95": 0.1247347816824913 }, { "label": "completed_group_5", "mean": 0.12607695162296295, "median": 0.1261017620563507, "p05": 0.12524889409542084, "p95": 0.12690351903438568 }, { "label": "completed_group_6", "mean": 0.127593994140625, "median": 0.12773489952087402, "p05": 0.1267755776643753, "p95": 0.1284082531929016 }, { "label": "current_group_partial", "mean": 0.12782885134220123, "median": 0.1278689205646515, "p05": 0.12723052501678467, "p95": 0.12838256359100342 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.073427200317383, "group": 7, "latest_source_mass": 0.10513068735599518, "layer": 27, "max_source_mass": 0.1465209424495697, "mixer_index": 53, "normalized_entropy": 0.9971077131807252, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.13135746121406555, "median": 0.13279247283935547, "p05": 0.11876361072063446, "p95": 0.14146237075328827 }, { "label": "completed_group_1", "mean": 0.1465209424495697, "median": 0.1469956636428833, "p05": 0.13593894243240356, "p95": 0.1559162735939026 }, { "label": "completed_group_2", "mean": 0.13772746920585632, "median": 0.137643963098526, "p05": 0.13452906906604767, "p95": 0.14171628654003143 }, { "label": "completed_group_3", "mean": 0.12966199219226837, "median": 0.1287793666124344, "p05": 0.12537093460559845, "p95": 0.13508695363998413 }, { "label": "completed_group_4", "mean": 0.12215200066566467, "median": 0.12143252789974213, "p05": 0.11671329289674759, "p95": 0.129044771194458 }, { "label": "completed_group_5", "mean": 0.11654195189476013, "median": 0.11616532504558563, "p05": 0.11162350326776505, "p95": 0.12263412773609161 }, { "label": "completed_group_6", "mean": 0.11090746521949768, "median": 0.11034701764583588, "p05": 0.10647446662187576, "p95": 0.11587852239608765 }, { "label": "current_group_partial", "mean": 0.10513068735599518, "median": 0.10488355159759521, "p05": 0.10140107572078705, "p95": 0.10828588157892227 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.079422950744629, "group": 7, "latest_source_mass": 0.12521177530288696, "layer": 28, "max_source_mass": 0.12591606378555298, "mixer_index": 54, "normalized_entropy": 0.9999910596499906, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.12477636337280273, "median": 0.12485313415527344, "p05": 0.12292245030403137, "p95": 0.12664391100406647 }, { "label": "completed_group_1", "mean": 0.12493627518415451, "median": 0.12481105327606201, "p05": 0.12430030107498169, "p95": 0.12584592401981354 }, { "label": "completed_group_2", "mean": 0.1242394968867302, "median": 0.12431945651769638, "p05": 0.12357322871685028, "p95": 0.12491215020418167 }, { "label": "completed_group_3", "mean": 0.12470392882823944, "median": 0.12473497539758682, "p05": 0.12409180402755737, "p95": 0.12546151876449585 }, { "label": "completed_group_4", "mean": 0.12560302019119263, "median": 0.12554584443569183, "p05": 0.12500521540641785, "p95": 0.12630382180213928 }, { "label": "completed_group_5", "mean": 0.12461308389902115, "median": 0.12459056079387665, "p05": 0.12410678714513779, "p95": 0.12512892484664917 }, { "label": "completed_group_6", "mean": 0.12591606378555298, "median": 0.12592178583145142, "p05": 0.12557773292064667, "p95": 0.12638571858406067 }, { "label": "current_group_partial", "mean": 0.12521177530288696, "median": 0.12524773180484772, "p05": 0.12447575479745865, "p95": 0.1259688436985016 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.0725789070129395, "group": 7, "latest_source_mass": 0.10425455868244171, "layer": 28, "max_source_mass": 0.14827941358089447, "mixer_index": 55, "normalized_entropy": 0.9966997703328787, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.13186156749725342, "median": 0.13315711915493011, "p05": 0.11918869614601135, "p95": 0.14256390929222107 }, { "label": "completed_group_1", "mean": 0.14827941358089447, "median": 0.14931634068489075, "p05": 0.13603198528289795, "p95": 0.15790212154388428 }, { "label": "completed_group_2", "mean": 0.13866659998893738, "median": 0.13880926370620728, "p05": 0.1348804384469986, "p95": 0.14264559745788574 }, { "label": "completed_group_3", "mean": 0.12988534569740295, "median": 0.12922212481498718, "p05": 0.12565752863883972, "p95": 0.13522912561893463 }, { "label": "completed_group_4", "mean": 0.12183371186256409, "median": 0.12081405520439148, "p05": 0.1163119301199913, "p95": 0.12904007732868195 }, { "label": "completed_group_5", "mean": 0.11524046957492828, "median": 0.1147729679942131, "p05": 0.11042807996273041, "p95": 0.12173305451869965 }, { "label": "completed_group_6", "mean": 0.1099783331155777, "median": 0.10993096977472305, "p05": 0.10556004196405411, "p95": 0.11569196730852127 }, { "label": "current_group_partial", "mean": 0.10425455868244171, "median": 0.10401678085327148, "p05": 0.10061369091272354, "p95": 0.10750719904899597 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.0793380737304688, "group": 8, "latest_source_mass": 0.12683480978012085, "layer": 29, "max_source_mass": 0.12747105956077576, "mixer_index": 56, "normalized_entropy": 0.9999502424341857, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.12417487800121307, "median": 0.1240256130695343, "p05": 0.12294038385152817, "p95": 0.1256686896085739 }, { "label": "completed_group_1", "mean": 0.1225123256444931, "median": 0.12230424582958221, "p05": 0.12168098986148834, "p95": 0.12367479503154755 }, { "label": "completed_group_2", "mean": 0.12277349829673767, "median": 0.1226874440908432, "p05": 0.12208077311515808, "p95": 0.12351377308368683 }, { "label": "completed_group_3", "mean": 0.1243998259305954, "median": 0.12447299808263779, "p05": 0.12367013841867447, "p95": 0.12505075335502625 }, { "label": "completed_group_4", "mean": 0.12567630410194397, "median": 0.12574473023414612, "p05": 0.12491010874509811, "p95": 0.1263245940208435 }, { "label": "completed_group_5", "mean": 0.12615728378295898, "median": 0.12627410888671875, "p05": 0.1253039538860321, "p95": 0.1267564296722412 }, { "label": "completed_group_6", "mean": 0.12747105956077576, "median": 0.12754422426223755, "p05": 0.1268254816532135, "p95": 0.1280263066291809 }, { "label": "completed_group_7", "mean": 0.12683480978012085, "median": 0.12676092982292175, "p05": 0.1260336935520172, "p95": 0.12750548124313354 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.1906280517578125, "group": 8, "latest_source_mass": 0.09856453537940979, "layer": 29, "max_source_mass": 0.13291704654693604, "mixer_index": 57, "normalized_entropy": 0.9969977918295433, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.11854428797960281, "median": 0.11933885514736176, "p05": 0.10912582278251648, "p95": 0.12757083773612976 }, { "label": "completed_group_1", "mean": 0.13291704654693604, "median": 0.13307878375053406, "p05": 0.12339349836111069, "p95": 0.14226418733596802 }, { "label": "completed_group_2", "mean": 0.12378417700529099, "median": 0.1235300600528717, "p05": 0.12091188877820969, "p95": 0.1279762238264084 }, { "label": "completed_group_3", "mean": 0.11745279282331467, "median": 0.11693152785301208, "p05": 0.11398620158433914, "p95": 0.12128151953220367 }, { "label": "completed_group_4", "mean": 0.11140252649784088, "median": 0.11079759150743484, "p05": 0.10681547224521637, "p95": 0.11677644401788712 }, { "label": "completed_group_5", "mean": 0.10409101843833923, "median": 0.1040785163640976, "p05": 0.09952308982610703, "p95": 0.10878291726112366 }, { "label": "completed_group_6", "mean": 0.09838292002677917, "median": 0.09840121120214462, "p05": 0.09399936348199844, "p95": 0.10255295783281326 }, { "label": "completed_group_7", "mean": 0.09486069530248642, "median": 0.09467358142137527, "p05": 0.09144656360149384, "p95": 0.09743223339319229 }, { "label": "current_group_partial", "mean": 0.09856453537940979, "median": 0.09877555072307587, "p05": 0.09461960196495056, "p95": 0.10164139419794083 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.197185516357422, "group": 8, "latest_source_mass": 0.11096570640802383, "layer": 30, "max_source_mass": 0.11249847710132599, "mixer_index": 58, "normalized_entropy": 0.999982222582434, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.11084984987974167, "median": 0.11077362298965454, "p05": 0.108895443379879, "p95": 0.11311701685190201 }, { "label": "completed_group_1", "mean": 0.11021924018859863, "median": 0.11018644273281097, "p05": 0.10941747575998306, "p95": 0.11104057729244232 }, { "label": "completed_group_2", "mean": 0.11018813401460648, "median": 0.11026144027709961, "p05": 0.10942888259887695, "p95": 0.1108226552605629 }, { "label": "completed_group_3", "mean": 0.11046157777309418, "median": 0.11042240262031555, "p05": 0.10973355174064636, "p95": 0.11123604327440262 }, { "label": "completed_group_4", "mean": 0.11131839454174042, "median": 0.11132219433784485, "p05": 0.1105581521987915, "p95": 0.11199404299259186 }, { "label": "completed_group_5", "mean": 0.11167056858539581, "median": 0.1118055060505867, "p05": 0.11075222492218018, "p95": 0.1122528463602066 }, { "label": "completed_group_6", "mean": 0.11249847710132599, "median": 0.11257164180278778, "p05": 0.11187897622585297, "p95": 0.11291356384754181 }, { "label": "completed_group_7", "mean": 0.1118280440568924, "median": 0.11168843507766724, "p05": 0.11146453768014908, "p95": 0.11255130171775818 }, { "label": "current_group_partial", "mean": 0.11096570640802383, "median": 0.11086957156658173, "p05": 0.11044884473085403, "p95": 0.11181740462779999 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.1899101734161377, "group": 8, "latest_source_mass": 0.09411308169364929, "layer": 30, "max_source_mass": 0.13352498412132263, "mixer_index": 59, "normalized_entropy": 0.9966710713162742, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1190595030784607, "median": 0.11995811760425568, "p05": 0.10872812569141388, "p95": 0.1278010904788971 }, { "label": "completed_group_1", "mean": 0.13352498412132263, "median": 0.13314831256866455, "p05": 0.12346142530441284, "p95": 0.14397428929805756 }, { "label": "completed_group_2", "mean": 0.1239522397518158, "median": 0.12362635135650635, "p05": 0.12061451375484467, "p95": 0.12857040762901306 }, { "label": "completed_group_3", "mean": 0.11779220402240753, "median": 0.11750936508178711, "p05": 0.11429227143526077, "p95": 0.12138796597719193 }, { "label": "completed_group_4", "mean": 0.11234382539987564, "median": 0.11192750930786133, "p05": 0.1077679693698883, "p95": 0.11798053979873657 }, { "label": "completed_group_5", "mean": 0.10513705015182495, "median": 0.1050291359424591, "p05": 0.10075211524963379, "p95": 0.11021918058395386 }, { "label": "completed_group_6", "mean": 0.09944089502096176, "median": 0.09912091493606567, "p05": 0.09499282389879227, "p95": 0.10404862463474274 }, { "label": "completed_group_7", "mean": 0.09463620185852051, "median": 0.09429345279932022, "p05": 0.090972401201725, "p95": 0.0975509062409401 }, { "label": "current_group_partial", "mean": 0.09411308169364929, "median": 0.09450676292181015, "p05": 0.09012747555971146, "p95": 0.0978383868932724 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.197190284729004, "group": 8, "latest_source_mass": 0.11225657165050507, "layer": 31, "max_source_mass": 0.11242865771055222, "mixer_index": 60, "normalized_entropy": 0.9999843927618645, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.11067462712526321, "median": 0.11063650250434875, "p05": 0.10940495133399963, "p95": 0.11191637068986893 }, { "label": "completed_group_1", "mean": 0.11021950840950012, "median": 0.11016013473272324, "p05": 0.10925821959972382, "p95": 0.11120235174894333 }, { "label": "completed_group_2", "mean": 0.11038416624069214, "median": 0.1102733239531517, "p05": 0.10960154980421066, "p95": 0.11125709116458893 }, { "label": "completed_group_3", "mean": 0.11103622615337372, "median": 0.11098648607730865, "p05": 0.11065587401390076, "p95": 0.11149194091558456 }, { "label": "completed_group_4", "mean": 0.11016052961349487, "median": 0.11016929149627686, "p05": 0.10976231843233109, "p95": 0.11053308844566345 }, { "label": "completed_group_5", "mean": 0.11144963651895523, "median": 0.11151091754436493, "p05": 0.11074185371398926, "p95": 0.11193440854549408 }, { "label": "completed_group_6", "mean": 0.11242865771055222, "median": 0.11247988045215607, "p05": 0.11164618283510208, "p95": 0.11297892034053802 }, { "label": "completed_group_7", "mean": 0.11139006912708282, "median": 0.11143708229064941, "p05": 0.11081651598215103, "p95": 0.1119559034705162 }, { "label": "current_group_partial", "mean": 0.11225657165050507, "median": 0.11227837204933167, "p05": 0.1117941215634346, "p95": 0.11280770599842072 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.1896841526031494, "group": 8, "latest_source_mass": 0.09325435757637024, "layer": 31, "max_source_mass": 0.13428261876106262, "mixer_index": 61, "normalized_entropy": 0.9965682048112662, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.11968795955181122, "median": 0.1202392578125, "p05": 0.11006385833024979, "p95": 0.12905627489089966 }, { "label": "completed_group_1", "mean": 0.13428261876106262, "median": 0.13435032963752747, "p05": 0.12363448739051819, "p95": 0.14427264034748077 }, { "label": "completed_group_2", "mean": 0.12373493611812592, "median": 0.12346962094306946, "p05": 0.12061072885990143, "p95": 0.1282179057598114 }, { "label": "completed_group_3", "mean": 0.1179073229432106, "median": 0.11775766313076019, "p05": 0.11464057117700577, "p95": 0.12117566168308258 }, { "label": "completed_group_4", "mean": 0.11132999509572983, "median": 0.11077313870191574, "p05": 0.10667262226343155, "p95": 0.1171092763543129 }, { "label": "completed_group_5", "mean": 0.10504785180091858, "median": 0.1049019992351532, "p05": 0.10071787983179092, "p95": 0.10987957566976547 }, { "label": "completed_group_6", "mean": 0.09994535893201828, "median": 0.09992853552103043, "p05": 0.09573008865118027, "p95": 0.1044701635837555 }, { "label": "completed_group_7", "mean": 0.09480959922075272, "median": 0.09453058242797852, "p05": 0.09107209742069244, "p95": 0.09781570732593536 }, { "label": "current_group_partial", "mean": 0.09325435757637024, "median": 0.09382589161396027, "p05": 0.08936256170272827, "p95": 0.09653929620981216 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.1971938610076904, "group": 8, "latest_source_mass": 0.10965746641159058, "layer": 32, "max_source_mass": 0.11217275261878967, "mixer_index": 62, "normalized_entropy": 0.9999860203964374, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.11116788536310196, "median": 0.11119179427623749, "p05": 0.11027558147907257, "p95": 0.11194087564945221 }, { "label": "completed_group_1", "mean": 0.1117495447397232, "median": 0.11171998083591461, "p05": 0.11123906075954437, "p95": 0.1124686524271965 }, { "label": "completed_group_2", "mean": 0.11143069714307785, "median": 0.11137667298316956, "p05": 0.11108753085136414, "p95": 0.1119472086429596 }, { "label": "completed_group_3", "mean": 0.11117202788591385, "median": 0.11118572950363159, "p05": 0.11080538481473923, "p95": 0.11158231645822525 }, { "label": "completed_group_4", "mean": 0.11217275261878967, "median": 0.11215600371360779, "p05": 0.1119328960776329, "p95": 0.11246690899133682 }, { "label": "completed_group_5", "mean": 0.11135637760162354, "median": 0.11132287979125977, "p05": 0.11110222339630127, "p95": 0.1116970106959343 }, { "label": "completed_group_6", "mean": 0.11160033196210861, "median": 0.11164811253547668, "p05": 0.11120022088289261, "p95": 0.111964151263237 }, { "label": "completed_group_7", "mean": 0.10969290882349014, "median": 0.10973770171403885, "p05": 0.10930771380662918, "p95": 0.10996787250041962 }, { "label": "current_group_partial", "mean": 0.10965746641159058, "median": 0.10969755053520203, "p05": 0.10922328382730484, "p95": 0.1099214255809784 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.188610792160034, "group": 8, "latest_source_mass": 0.09157620370388031, "layer": 32, "max_source_mass": 0.1360917091369629, "mixer_index": 63, "normalized_entropy": 0.9960796974214496, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.11987313628196716, "median": 0.12083137035369873, "p05": 0.10883265733718872, "p95": 0.12953230738639832 }, { "label": "completed_group_1", "mean": 0.1360917091369629, "median": 0.13644108176231384, "p05": 0.12369051575660706, "p95": 0.14713600277900696 }, { "label": "completed_group_2", "mean": 0.12462760508060455, "median": 0.12453563511371613, "p05": 0.12107435613870621, "p95": 0.12926824390888214 }, { "label": "completed_group_3", "mean": 0.11755925416946411, "median": 0.11717041581869125, "p05": 0.11395573616027832, "p95": 0.12139421701431274 }, { "label": "completed_group_4", "mean": 0.11237476021051407, "median": 0.11172428727149963, "p05": 0.10730767995119095, "p95": 0.11879656463861465 }, { "label": "completed_group_5", "mean": 0.10438287258148193, "median": 0.10406170785427094, "p05": 0.09989674389362335, "p95": 0.10985209792852402 }, { "label": "completed_group_6", "mean": 0.09920359402894974, "median": 0.09911085665225983, "p05": 0.0946069061756134, "p95": 0.10452912747859955 }, { "label": "completed_group_7", "mean": 0.09431086480617523, "median": 0.09389802813529968, "p05": 0.09041668474674225, "p95": 0.09788760542869568 }, { "label": "current_group_partial", "mean": 0.09157620370388031, "median": 0.09197241812944412, "p05": 0.0875491350889206, "p95": 0.09489993005990982 } ], "sources": 9 }, { "branch": "output", "entropy_mean": 2.1930694580078125, "group": null, "latest_source_mass": 0.11946354806423187, "layer": null, "max_source_mass": 0.12094368040561676, "mixer_index": null, "normalized_entropy": 0.9981089236979843, "offset": null, "source_summaries": [ { "label": "embedding", "mean": 0.10071488469839096, "median": 0.09996568411588669, "p05": 0.09106086939573288, "p95": 0.1130409762263298 }, { "label": "completed_group_1", "mean": 0.09411877393722534, "median": 0.09266924858093262, "p05": 0.08850140869617462, "p95": 0.10204548388719559 }, { "label": "completed_group_2", "mean": 0.10181896388530731, "median": 0.10170605033636093, "p05": 0.09935513883829117, "p95": 0.10529708862304688 }, { "label": "completed_group_3", "mean": 0.10989432036876678, "median": 0.11005622148513794, "p05": 0.1059446707367897, "p95": 0.11359670758247375 }, { "label": "completed_group_4", "mean": 0.1145806610584259, "median": 0.11529573798179626, "p05": 0.10914603620767593, "p95": 0.11935751885175705 }, { "label": "completed_group_5", "mean": 0.11861918121576309, "median": 0.11936421692371368, "p05": 0.11333125829696655, "p95": 0.12279194593429565 }, { "label": "completed_group_6", "mean": 0.11984598636627197, "median": 0.12033014744520187, "p05": 0.11644290387630463, "p95": 0.12263244390487671 }, { "label": "completed_group_7", "mean": 0.12094368040561676, "median": 0.12095880508422852, "p05": 0.118732750415802, "p95": 0.12314030528068542 }, { "label": "completed_group_8", "mean": 0.11946354806423187, "median": 0.118886798620224, "p05": 0.11734164506196976, "p95": 0.12209592759609222 } ], "sources": 9 } ], "mode": "learned", "optimizer_state_after": "76e623d010bd259bd22b2b58ea23757691a34f06b3d4e494d9fed2eb66191714", "optimizer_state_before": "76e623d010bd259bd22b2b58ea23757691a34f06b3d4e494d9fed2eb66191714", "optimizer_state_unchanged": true, "positions": { "attention_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "19992b8c09cd63e9f590e656df98ebaa4fbdc484ab53c23d4c73169c5f0396ea", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.002919268561527133, 0.0026904908008873463, 0.0028405722696334124, 0.0029144894797354937, 0.0030634324066340923, 0.0032732728868722916, 0.003848863998427987, 0.003392934100702405, 0.0037359180860221386, 0.004302032291889191, 0.004344160668551922, 0.004835603758692741, 0.005531937349587679, 0.005483300890773535, 0.005397035740315914, 0.005739198997616768, 0.005659298039972782, 0.007230010814964771, 0.006938732694834471, 0.007343437522649765, 0.007882392033934593, 0.00830139685422182, 0.008992243558168411, 0.009393857792019844, 0.012831038795411587, 0.00905293133109808, 0.012243949808180332, 0.012258118949830532, 0.013431218452751637, 0.01483316533267498, 0.015388047322630882, 0.014634119346737862 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.570841959619429e-05, "first_to_last_ratio": 3.472594525431938, "imbalance_abs_log_ratio": 1.2449020164225795, "last_quartile_mean": 4.523539814726973e-06, "mean": 8.683279858701098e-06, "normalized": [ 2.229894338065868, 2.1043034837063286, 1.976946278509285, 1.8612974320657731, 1.7187480804487953, 1.6201790212455152, 1.5254325865776683, 1.4355362848487128, 1.3017373037261373, 1.2100830304331005, 1.133220479031311, 1.0535486714736453, 0.9690659040803394, 0.9036010498699347, 0.8438110062145913, 0.7926660684312774, 0.749751881449241, 0.7082035160351038, 0.6752208628773162, 0.6475764877057508, 0.6244377002240272, 0.6007972561165787, 0.5813749906205735, 0.5649799471303525, 0.5550363681005585, 0.5437815898914894, 0.5335487688172357, 0.5248093961192007, 0.5123845047471113, 0.5051729354463279, 0.4992577444849533, 0.49359503150589623 ], "peak_layer": 1, "peak_normalized": 2.229894338065868, "population_cv": 0.5276859177486317, "rest_mean": 9.350082985259144e-06, "spike_contrast": 0.543582659458801, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.082542975287651e-06, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 1.936279659275897e-05, 1.8272256056661718e-05, 1.7166377801913768e-05, 1.6162166502908804e-05, 1.49243705891422e-05, 1.4068467862671241e-05, 1.3245758054836188e-05, 1.2465163308661431e-05, 1.1303349310765043e-05, 1.050748960551573e-05, 9.840070561040193e-06, 9.148257959168404e-06, 8.414670446654782e-06, 7.846220796636771e-06, 7.3270471148134675e-06, 6.8829413066850975e-06, 6.510305411211448e-06, 6.149529326648917e-06, 5.863131718797376e-06, 5.623087872663746e-06, 5.42216730536893e-06, 5.216890713199973e-06, 5.048241746408166e-06, 4.905878995487001e-06, 4.819536115974188e-06, 4.7218077270372305e-06, 4.632953277905472e-06, 4.557066858978942e-06, 4.4491780499811284e-06, 4.38655797552201e-06, 4.335194716986734e-06, 4.286023795430083e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 6.106548153184121e-05, "first_to_last_ratio": 3.396664379322294, "imbalance_abs_log_ratio": 1.2227938851577476, "last_quartile_mean": 1.7978073401536676e-05, "mean": 3.421872185072061e-05, "normalized": [ 2.125690241222297, 2.035753111677548, 1.9424653444151043, 1.8443604241196392, 1.713002582679694, 1.6251047960941019, 1.5375111780105508, 1.4526202049778152, 1.3158411261187606, 1.226794175835664, 1.149073116741286, 1.0695301297827082, 0.9835511244361359, 0.9163244633971447, 0.8552224227935168, 0.8029396208596036, 0.7578907347648619, 0.7156154592837152, 0.6817576032295484, 0.6538669675872317, 0.6299332335154186, 0.605919603383038, 0.5863501185278558, 0.569785799336057, 0.5595323652477712, 0.5483698809054829, 0.5380511691309848, 0.5293393622233508, 0.5166837305455455, 0.5094821346031305, 0.5036034749610113, 0.498034299593426 ], "peak_layer": 1, "peak_normalized": 2.125690241222297, "population_cv": 0.5121606369233174, "rest_mean": 3.681488218447797e-05, "spike_contrast": 0.5486763735168884, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.0199456048430876e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 7.273840310517699e-05, 6.966086948523298e-05, 6.646868132520467e-05, 6.311165634542704e-05, 5.8616758906282485e-05, 5.560900899581611e-05, 5.261166734271683e-05, 4.970680674887262e-05, 4.502640149439685e-05, 4.197932867100462e-05, 3.931981336791068e-05, 3.659795402199961e-05, 3.365586235304363e-05, 3.1355451937997714e-05, 2.9264618206070736e-05, 2.747556754911784e-05, 2.593405224615708e-05, 2.448744635330513e-05, 2.3328873794525862e-05, 2.2374491891241632e-05, 2.1555510102189146e-05, 2.073379437206313e-05, 2.006415161304176e-05, 1.9497341781971045e-05, 1.9146482372889295e-05, 1.8764516426017508e-05, 1.8411423297948204e-05, 1.8113316400558688e-05, 1.76802568603307e-05, 1.7433827451895922e-05, 1.723266723274719e-05, 1.70420971699059e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.054153479635715485, "first_to_last_ratio": 3.3966644557946295, "imbalance_abs_log_ratio": 1.2227939076716985, "last_quartile_mean": 0.01594313490204513, "mean": 0.030345504521392286, "normalized": [ 2.1256903113360446, 2.035753146214189, 1.9424653011478115, 1.8443605007189683, 1.71300252819031, 1.6251048166450035, 1.5375111878366567, 1.452620175683203, 1.3158411334809852, 1.2267941193182312, 1.1490731743165223, 1.0695301881432524, 0.9835511318647784, 0.9163244786384582, 0.8552223856485347, 0.8029396127492687, 0.7578907435068827, 0.7156154476500971, 0.6817576088078939, 0.6538669486644281, 0.6299332311205004, 0.6059195951826442, 0.586350084864764, 0.569785800788618, 0.5595323690576035, 0.5483698809409665, 0.5380511395546987, 0.5293393594226915, 0.5166837108984478, 0.5094821244722924, 0.5036034882338023, 0.4980342749014515 ], "peak_layer": 1, "peak_normalized": 2.1256903113360446, "population_cv": 0.5121606472523333, "rest_mean": 0.0326478054953946, "spike_contrast": 0.5486763655311122, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.017913079261779784, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.06450514495372772, 0.06177595630288124, 0.05894508957862854, 0.05596804991364479, 0.05198192596435547, 0.049314625561237335, 0.04665655270218849, 0.04408049210906029, 0.039929863065481186, 0.03722768649458885, 0.034869205206632614, 0.032455433160066605, 0.029846355319023132, 0.027806328609585762, 0.025952154770493507, 0.024365607649087906, 0.02299857698380947, 0.021715711802244186, 0.02068827860057354, 0.01984192244708538, 0.019115641713142395, 0.01838693581521511, 0.017793089151382446, 0.01729043759405613, 0.016979292035102844, 0.01664056070148945, 0.01632743328809738, 0.01606306992471218, 0.015679027885198593, 0.015460492111742496, 0.015282101929187775, 0.015113101340830326 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 5.484954840540013e-06, "first_to_last_ratio": 4.317118296735341, "imbalance_abs_log_ratio": 1.462588118730986, "last_quartile_mean": 1.2705129819323702e-06, "mean": 2.775344690064685e-06, "normalized": [ 2.642311887415898, 2.416230245995366, 2.21143682159549, 2.0333492354809493, 1.8430856012839139, 1.6939904160227164, 1.5446558421616008, 1.4254585070372157, 1.2827451334693212, 1.1755524566426843, 1.0923958043799127, 1.0016640021807137, 0.9226630606403423, 0.8576190807297013, 0.7936412075441582, 0.7370849544400642, 0.6937134258189417, 0.6483319177623661, 0.6136592809430983, 0.5846488972705621, 0.5617626261670844, 0.5391223387293338, 0.5197976899046192, 0.5027943403920482, 0.49023336166728243, 0.4787835081603097, 0.4696409445710295, 0.46085508801940983, 0.450168541694312, 0.44355065941556593, 0.43718887869853934, 0.4318642437654502 ], "peak_layer": 1, "peak_normalized": 2.642311887415898, "population_cv": 0.6359571845984164, "rest_mean": 3.020632700807053e-06, "spike_contrast": 0.48029322852403633, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.450789432055899e-06, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 7.333326266234508e-06, 6.705871783196926e-06, 6.1374994402285665e-06, 5.643245003739139e-06, 5.1151978368579876e-06, 4.701407306129113e-06, 4.286952389520593e-06, 3.95613869841327e-06, 3.5600598948803963e-06, 3.2625632684357697e-06, 3.031774895134731e-06, 2.779962869681185e-06, 2.5607080260670045e-06, 2.380188561801333e-06, 2.202627911174204e-06, 2.0456648144318024e-06, 1.9252938727731816e-06, 1.7993445453612367e-06, 1.7031160268743406e-06, 1.622602212592028e-06, 1.5590849216096103e-06, 1.4962503200877109e-06, 1.4426177585846744e-06, 1.3954276028016466e-06, 1.3605665571958525e-06, 1.3287892670632573e-06, 1.3034155017521698e-06, 1.2790317214239622e-06, 1.2493728718254715e-06, 1.2310059673836804e-06, 1.2133498330513248e-06, 1.1985721357632428e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 5.000926375942072e-05, "first_to_last_ratio": 2.861076547442411, "imbalance_abs_log_ratio": 1.0511979691949909, "last_quartile_mean": 1.7479177131463075e-05, "mean": 3.001758557275025e-05, "normalized": [ 1.966517821002492, 1.8844494706669952, 1.8016591622079403, 1.7146543004927615, 1.602713342111759, 1.525842314618408, 1.4518503935852287, 1.3803041859872354, 1.2718526716709437, 1.1969100977048808, 1.1317671023781757, 1.06503487809827, 0.9973441232158428, 0.941363281859895, 0.8896494105618908, 0.8445592656875696, 0.8090600018046681, 0.7713325693634966, 0.7401533589124044, 0.7139326567185207, 0.692297727813069, 0.6685034690624881, 0.6485427267474059, 0.6313224439068734, 0.6206897714046083, 0.6084823527417841, 0.5970360381608709, 0.5872159169829965, 0.572597446631392, 0.564250570870669, 0.5573663965692749, 0.5507447304591896 ], "peak_layer": 1, "peak_normalized": 1.966517821002492, "population_cv": 0.43784911874358545, "rest_mean": 3.1950544494234406e-05, "spike_contrast": 0.6128098192589182, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.957960739673581e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 5.9030116972280666e-05, 5.656662324327044e-05, 5.408145807450637e-05, 5.146978219272569e-05, 4.810958489542827e-05, 4.580210224958137e-05, 4.3581043428275734e-05, 4.143339901929721e-05, 3.817794640781358e-05, 3.592835128074512e-05, 3.397291584406048e-05, 3.196977559127845e-05, 2.993786256411113e-05, 2.8257452868274413e-05, 2.670512731128838e-05, 2.5351630029035732e-05, 2.4286027837661095e-05, 2.3153541405918077e-05, 2.2217616788111627e-05, 2.1430534616229124e-05, 2.0781106286449358e-05, 2.006686008826364e-05, 1.9467686797725037e-05, 1.8950775483972393e-05, 1.8631608327268623e-05, 1.8265171092934906e-05, 1.7921580365509726e-05, 1.7626804037718102e-05, 1.7187992852996103e-05, 1.6937439795583487e-05, 1.673079350439366e-05, 1.6532027075299993e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 3.89006590921781e-05, "first_to_last_ratio": 2.1665975586085744, "imbalance_abs_log_ratio": 0.7731579916979808, "last_quartile_mean": 1.7954723034563358e-05, "mean": 2.587040432899812e-05, "normalized": [ 1.7831448102947671, 1.6974226005892297, 1.620036533641332, 1.533717800359919, 1.4465980773830347, 1.3745334732258696, 1.316859965180907, 1.257080202982369, 1.1876570648434144, 1.1203272435244702, 1.0718495078032313, 1.0193313865605265, 0.978296344153428, 0.937610047259807, 0.9066038289594328, 0.877320807240572, 0.8649781691504772, 0.8401000985755385, 0.8145599740177561, 0.792555397726043, 0.7795027531450014, 0.7587311609912605, 0.7416782788097599, 0.7272996278499536, 0.7222997702968516, 0.7135070943498951, 0.7058159198574021, 0.6988176661381593, 0.6867067750666979, 0.6814848031835403, 0.675082580859005, 0.668490235980348 ], "peak_layer": 1, "peak_normalized": 1.7831448102947671, "population_cv": 0.3297379118959793, "rest_mean": 2.708773947098827e-05, "spike_contrast": 0.7123811340152144, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.9296794562251307e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 4.6130677219480276e-05, 4.391300899442285e-05, 4.191100015304983e-05, 3.967789962189272e-05, 3.742407716345042e-05, 3.555973671609536e-05, 3.406769974390045e-05, 3.252117312513292e-05, 3.072516847169027e-05, 2.8983318770769984e-05, 2.7729180146707222e-05, 2.63705151155591e-05, 2.530892197682988e-05, 2.4256351025542244e-05, 2.3454207621398382e-05, 2.269664400955662e-05, 2.2377334971679375e-05, 2.173372922698036e-05, 2.1072995878057554e-05, 2.050372859230265e-05, 2.0166051399428397e-05, 1.9628681911854073e-05, 1.9187516954843886e-05, 1.8815535440808162e-05, 1.8686187104322016e-05, 1.8458717022440396e-05, 1.8259743228554726e-05, 1.8078695575241e-05, 1.776538192643784e-05, 1.7630287402425893e-05, 1.7464659322286025e-05, 1.729411269479897e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00011786452796513913, "first_to_last_ratio": 5.272419649365271, "imbalance_abs_log_ratio": 1.6624893936843081, "last_quartile_mean": 2.2354921611622558e-05, "mean": 5.981176059322024e-05, "normalized": [ 2.342046121512161, 2.2441163908348334, 2.142933559820905, 2.032574241487723, 1.8909432158278896, 1.7811012647720825, 1.7046454997674345, 1.6263691566595195, 1.4513535263674326, 1.3484633711734058, 1.2565665103260477, 1.1592512217173825, 1.0440274588303153, 0.9501757209203151, 0.8639054268917737, 0.7923577171085253, 0.7123005746652016, 0.6458221031841243, 0.5963034371094506, 0.5569297557861542, 0.5166112374905155, 0.4756772431215453, 0.44740588281370386, 0.42808243396944073, 0.41207290926584317, 0.3977052623291704, 0.38540407372369656, 0.3748332006962577, 0.3630559483389979, 0.3565727390637327, 0.35157439090519904, 0.34881840351921967 ], "peak_layer": 1, "peak_normalized": 2.342046121512161, "population_cv": 0.6458102290367006, "rest_mean": 6.583757459776718e-05, "spike_contrast": 0.41423708475420845, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.7272364968666805e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00014008190191816539, 0.00013422455231193453, 0.00012817262904718518, 0.00012157184391980991, 0.00011310064292047173, 0.00010653080244082958, 0.00010195784852840006, 9.727600263431668e-05, 8.680800965521485e-05, 8.065396832535043e-05, 7.515745528507978e-05, 6.933685654075816e-05, 6.244512042030692e-05, 5.683168274117634e-05, 5.167170456843451e-05, 4.739231007988565e-05, 4.2603951442288235e-05, 3.862775702145882e-05, 3.566595842130482e-05, 3.331094922032207e-05, 3.089942765654996e-05, 2.8451093385228887e-05, 2.6760133550851606e-05, 2.56043640547432e-05, 2.464680619596038e-05, 2.3787451937096193e-05, 2.3051696189213544e-05, 2.241943366243504e-05, 2.171501546399668e-05, 2.1327243302948773e-05, 2.1028283299528994e-05, 2.086344284180086e-05 ] } } }, "mlp_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "bddd76f023c30d419ef37ac99b99ff2b543a470b0e0414eb98403c83e399da9b", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.005079356953501701, 0.0052208867855370045, 0.005689430516213179, 0.005868426524102688, 0.00590060418471694, 0.0059102908708155155, 0.0058776624500751495, 0.006663382984697819, 0.006277414970099926, 0.006413500756025314, 0.006856496445834637, 0.006499431096017361, 0.006718362681567669, 0.0070298356004059315, 0.006969383917748928, 0.0068367342464625835, 0.007033169735223055, 0.007157569285482168, 0.0075136395171284676, 0.007708695251494646, 0.007887008599936962, 0.008205465972423553, 0.00814912747591734, 0.008287571370601654, 0.008026733994483948, 0.008120592683553696, 0.008479002863168716, 0.00907627958804369, 0.00841403380036354, 0.008773921057581902, 0.008821850642561913, 0.009164483286440372 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.5008465311439068e-05, "first_to_last_ratio": 3.3668968034019833, "imbalance_abs_log_ratio": 1.213991490288974, "last_quartile_mean": 4.457655279566097e-06, "mean": 8.299250268350988e-06, "normalized": [ 2.2568151540450523, 2.1152935419859507, 1.9767562187382288, 1.86698957878165, 1.7143188557206628, 1.6128346254213421, 1.5135326124803135, 1.410756892623042, 1.2736883716277778, 1.1902867485409472, 1.1070374521817832, 1.0322313304507316, 0.9480382659054776, 0.8848529200942086, 0.8311995014573598, 0.7835740569083032, 0.742040914352895, 0.7071972712748903, 0.6782348645232779, 0.6514608668278806, 0.6287747596064291, 0.6086256047198574, 0.5913261103101772, 0.5772101349211559, 0.569144818247716, 0.5582752103529628, 0.549190400479895, 0.5409965375823161, 0.5285413070813697, 0.5224600186388942, 0.5164682084504763, 0.5118468456669756 ], "peak_layer": 1, "peak_normalized": 2.2568151540450523, "population_cv": 0.524209711679407, "rest_mean": 8.921669009227858e-06, "spike_contrast": 0.5535050742757021, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.9381890676158944e-06, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 1.8729873772826977e-05, 1.7555350495968014e-05, 1.640559457882773e-05, 1.5494613762712106e-05, 1.422756122337887e-05, 1.338531819783384e-05, 1.2561185940285213e-05, 1.1708224519679789e-05, 1.0570658560027368e-05, 9.878487617243081e-06, 9.187580872094259e-06, 8.566746146243531e-06, 7.86800683272304e-06, 7.343615834543016e-06, 6.898332685523201e-06, 6.5030772020691074e-06, 6.1583832575706765e-06, 5.8692071434052195e-06, 5.62884088139981e-06, 5.406636773841456e-06, 5.2183590923959855e-06, 5.051136213296559e-06, 4.907563379674684e-06, 4.790411367139313e-06, 4.723475285572931e-06, 4.633265689335531e-06, 4.557868578558555e-06, 4.489865659706993e-06, 4.38649658462964e-06, 4.336026449891506e-06, 4.286298917577369e-06, 4.247945071256254e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 5.820503929498955e-05, "first_to_last_ratio": 3.2848952430521337, "imbalance_abs_log_ratio": 1.1893347619462469, "last_quartile_mean": 1.7718994058668613e-05, "mean": 3.2650613661644456e-05, "normalized": [ 2.1450013507239705, 2.0461366500203595, 1.9398189683428497, 1.8522773613526717, 1.7094590145500226, 1.6168974254280155, 1.5255888819635426, 1.4261239538940118, 1.2879114356833505, 1.2060871976977126, 1.1221510774968064, 1.0481371490840103, 0.9613461884209538, 0.8968067603991513, 0.8421359861574166, 0.7933538356930859, 0.7503585255090798, 0.7148355279210235, 0.6855522988186872, 0.6585248527378349, 0.6351822812313331, 0.6146906504361358, 0.5972743074473386, 0.5828693520617939, 0.574802048908203, 0.5639397943233103, 0.554845965325609, 0.5465864044553121, 0.5340118249846835, 0.5278506086231465, 0.5219704528761507, 0.5174718674324269 ], "peak_layer": 1, "peak_normalized": 2.1450013507239705, "population_cv": 0.5072019698199706, "rest_mean": 3.5063350541922434e-05, "spike_contrast": 0.5596109386261622, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.9621834508143365e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 7.003561040619388e-05, 6.680761725874618e-05, 6.333627970889211e-05, 6.047799251973629e-05, 5.581488585448824e-05, 5.279269316815771e-05, 4.981141319149174e-05, 4.656382225221023e-05, 4.205109871691093e-05, 3.937948713428341e-05, 3.663892130134627e-05, 3.422232111915946e-05, 3.138854299322702e-05, 2.9281291062943637e-05, 2.749625673459377e-05, 2.5903489586198702e-05, 2.4499666324118152e-05, 2.3339818653766997e-05, 2.238370325358119e-05, 2.1501240553334355e-05, 2.0739091269206256e-05, 2.007002694881521e-05, 1.9501372662489302e-05, 1.903104202938266e-05, 1.8767639630823396e-05, 1.841298035287764e-05, 1.811606125556864e-05, 1.7846381524577737e-05, 1.7435813788324594e-05, 1.7234646293218248e-05, 1.704265559965279e-05, 1.6895774024305865e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.05161681119352579, "first_to_last_ratio": 3.284895287464527, "imbalance_abs_log_ratio": 1.1893347754664323, "last_quartile_mean": 0.015713380998931825, "mean": 0.028954890876775607, "normalized": [ 2.145001289725201, 2.046136691673054, 1.939818911776489, 1.8522774119626542, 1.7094589830823987, 1.6168973606656405, 1.5255889998464391, 1.4261239529824867, 1.287911366616444, 1.2060871893436458, 1.1221510842273104, 1.0481372445524282, 0.961346209259541, 0.8968067750917011, 0.8421359558072424, 0.7933538963986676, 0.7503585064155442, 0.7148355327095294, 0.6855522990290889, 0.6585248332975209, 0.6351822731624658, 0.6146906547931704, 0.5972742941320782, 0.5828693766065183, 0.5748020438293477, 0.5639398000251236, 0.5548459661026797, 0.5465864176570202, 0.534011826874325, 0.527850568480463, 0.5219704287182262, 0.5174718551555549 ], "peak_layer": 1, "peak_normalized": 2.145001289725201, "population_cv": 0.5072019680935582, "rest_mean": 0.03109453007992771, "spike_contrast": 0.5596109391274233, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.017400839179754258, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.06210827827453613, 0.05924566462635994, 0.05616724491119385, 0.05363249033689499, 0.049497198313474655, 0.04681708663702011, 0.044173263013362885, 0.04129326343536377, 0.037291333079338074, 0.034922122955322266, 0.032491762191057205, 0.030348699539899826, 0.027835674583911896, 0.025966942310333252, 0.02438395470380783, 0.02297147549688816, 0.021726548671722412, 0.020697984844446182, 0.019850092008709908, 0.019067514687776566, 0.01839163340628147, 0.017798300832509995, 0.017294012010097504, 0.01687691919505596, 0.016643330454826355, 0.01632881537079811, 0.016065504401922226, 0.015826350077986717, 0.015462254174053669, 0.015283855609595776, 0.015113596804440022, 0.014983341097831726 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 5.355004418561293e-06, "first_to_last_ratio": 4.2803271552883535, "imbalance_abs_log_ratio": 1.4540294448239433, "last_quartile_mean": 1.251073626917787e-06, "mean": 2.683941971781678e-06, "normalized": [ 2.7030485685097614, 2.4621902558382964, 2.2386850638453395, 2.05248673466923, 1.8575399210018237, 1.6957758104696088, 1.5437340831941542, 1.4081496862748333, 1.2505669189373825, 1.154757255828014, 1.0675003825206393, 0.9762526930125442, 0.897424076523062, 0.8318154411009656, 0.7746171026378299, 0.7211559397261628, 0.6781980327113627, 0.6399672113971655, 0.60846523760761, 0.5811474026512373, 0.5581318010721549, 0.5402081583654614, 0.5217772343602641, 0.5073418706694675, 0.49649585255921586, 0.4853123246471542, 0.4772675785905054, 0.4696153655827239, 0.458545908189773, 0.453049428909843, 0.4466290334907211, 0.4421476251056929 ], "peak_layer": 1, "peak_normalized": 2.7030485685097614, "population_cv": 0.6457571562979046, "rest_mean": 2.9201333467830814e-06, "spike_contrast": 0.48234391361810997, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.4085085467741009e-06, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 7.254825504787732e-06, 6.608375770156272e-06, 6.008500804455252e-06, 5.508755293703871e-06, 4.985529358236818e-06, 4.551363872451475e-06, 4.143292699154699e-06, 3.7793920455442276e-06, 3.3564490422577364e-06, 3.0993014661362395e-06, 2.8651090815401403e-06, 2.6202055778412614e-06, 2.4086341454676585e-06, 2.232544375146972e-06, 2.079027353829588e-06, 1.9355406948307063e-06, 1.82024416517379e-06, 1.7176348592329305e-06, 1.633085389585176e-06, 1.5597659057675628e-06, 1.4979933666836587e-06, 1.4498873497359455e-06, 1.4004198192196782e-06, 1.3616761407320155e-06, 1.332566057499207e-06, 1.3025501175434329e-06, 1.280958485949668e-06, 1.2604203902810696e-06, 1.2307106089792796e-06, 1.2159583775428473e-06, 1.198726408802031e-06, 1.1866985687447595e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 4.78605675198196e-05, "first_to_last_ratio": 2.7793570801926766, "imbalance_abs_log_ratio": 1.0222196348386712, "last_quartile_mean": 1.7220013887708774e-05, "mean": 2.881263026210945e-05, "normalized": [ 1.9752570579468534, 1.888104295535499, 1.7945647548145947, 1.7154982752112686, 1.5967511433618535, 1.5186186336526764, 1.441625524284827, 1.3583546018661343, 1.2494560102010674, 1.1811084039550475, 1.1110423537101946, 1.0496234480105349, 0.9817226877095458, 0.9275068527847158, 0.8805127756083984, 0.8381970075463177, 0.803976865779107, 0.7714567335637176, 0.7440882717080923, 0.7183088339322109, 0.696644002608348, 0.6758702747761651, 0.6578620296540808, 0.6426087873852322, 0.6340400523715064, 0.6220565885755293, 0.611876165213302, 0.6024355818665901, 0.5879093007530156, 0.5807453084805843, 0.5738005724905904, 0.5683768046423998 ], "peak_layer": 1, "peak_normalized": 1.9752570579468534, "population_cv": 0.4317079414337261, "rest_mean": 3.0619262057977416e-05, "spike_contrast": 0.622380073312624, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.905681856442243e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 5.691235128324479e-05, 5.440125096356496e-05, 5.1706130761886016e-05, 4.9428017518948764e-05, 4.6006600314285606e-05, 4.375539720058441e-05, 4.153702320763841e-05, 3.9137768908403814e-05, 3.600011405069381e-05, 3.4030839742626995e-05, 3.201205254299566e-05, 3.0242412321968004e-05, 2.8286012820899487e-05, 2.6723912014858797e-05, 2.5369889044668525e-05, 2.4150660465238616e-05, 2.3164688172983006e-05, 2.2227697627386078e-05, 2.14391402550973e-05, 2.069636684609577e-05, 2.0072146071470343e-05, 1.9473600332275964e-05, 1.8954735423903912e-05, 1.8515249394113198e-05, 1.8268361600348726e-05, 1.7923086488735862e-05, 1.7629761714488268e-05, 1.7357753677060828e-05, 1.6939213310251944e-05, 1.673279984970577e-05, 1.6532703739358112e-05, 1.637643072172068e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 3.7207858895271784e-05, "first_to_last_ratio": 2.0918563583707814, "imbalance_abs_log_ratio": 0.7380518814986469, "last_quartile_mean": 1.778700470822514e-05, "mean": 2.5014859602379147e-05, "normalized": [ 1.766896017685199, 1.6852067556838466, 1.5917675487924012, 1.5166567899677899, 1.4315811273281913, 1.3662079609937838, 1.303579546183775, 1.2375462607814267, 1.160212708426433, 1.1098915366489155, 1.0567965268051207, 1.0112629998208633, 0.9697402160962814, 0.9378324250878841, 0.907941204027948, 0.8814449641361121, 0.8690589692914835, 0.8427704073086626, 0.8201725639084954, 0.8014260695185076, 0.784571291095966, 0.7669834591017225, 0.7521317253856684, 0.7398605496651066, 0.7380371143547211, 0.7300178093260107, 0.722929928084616, 0.7169054508143122, 0.7047914878526057, 0.6980364293666598, 0.6913059490086015, 0.6864362074508885 ], "peak_layer": 1, "peak_normalized": 1.766896017685199, "population_cv": 0.3175074310788083, "rest_mean": 2.614369299812501e-05, "spike_contrast": 0.7236605504321191, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.8919159265351483e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 4.419865581439808e-05, 4.215521039441228e-05, 3.981784175266512e-05, 3.79389566660393e-05, 3.581080090953037e-05, 3.417550033191219e-05, 3.260885932832025e-05, 3.095704596489668e-05, 2.9022558010183275e-05, 2.776378096314147e-05, 2.6435616746312007e-05, 2.5296601961599663e-05, 2.4257915356429294e-05, 2.345974644413218e-05, 2.2712021745974198e-05, 2.2049222025088966e-05, 2.173938810301479e-05, 2.1081783415866084e-05, 2.051650153589435e-05, 2.0047560610692017e-05, 1.962594069482293e-05, 1.9185983546776697e-05, 1.8814469513017684e-05, 1.8507507775211707e-05, 1.846189479692839e-05, 1.8261293007526547e-05, 1.808399065339472e-05, 1.793328920030035e-05, 1.763026011758484e-05, 1.7461283277953044e-05, 1.7292921256739646e-05, 1.7171105355373584e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00011250984425714705, "first_to_last_ratio": 5.136972637567696, "imbalance_abs_log_ratio": 1.6364639249316602, "last_quartile_mean": 2.1901974605498253e-05, "mean": 5.6501408323583746e-05, "normalized": [ 2.382164866319327, 2.279581056980086, 2.1640865436402454, 2.0688186882965165, 1.8923495642236086, 1.8092596842626791, 1.7252927191320069, 1.6086472166206554, 1.4275667956980447, 1.3299589734449933, 1.2272752348601612, 1.1370747967113455, 1.007765000781145, 0.9145523912616853, 0.8390743254016101, 0.7615077898186029, 0.6843792173338624, 0.6320158419323604, 0.5900143308948815, 0.550751795487684, 0.5037684703042699, 0.4736161081951874, 0.4535521893538639, 0.4358391510862496, 0.4210578321323425, 0.40810041507267314, 0.3968839045479846, 0.3877734415065673, 0.37747110329431527, 0.3721870546903874, 0.36935648798288095, 0.3682570087317776 ], "peak_layer": 1, "peak_normalized": 2.382164866319327, "population_cv": 0.6517494715357345, "rest_mean": 6.217700879292183e-05, "spike_contrast": 0.41579944566424976, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.5853165789158082e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00013459566980600357, 0.00012879954010713845, 0.00012227393744979054, 0.00011689116945490241, 0.00010692041541915387, 0.00010222572018392384, 9.748146840138361e-05, 9.089083323488012e-05, 8.065953443292528e-05, 7.514455501222983e-05, 6.934277917025611e-05, 6.424632738344371e-05, 5.694014180335216e-05, 5.16734980919864e-05, 4.740888107335195e-05, 4.302626257413067e-05, 3.866838960675523e-05, 3.5709785151993856e-05, 3.3336640626657754e-05, 3.111825208179653e-05, 2.8463628041208722e-05, 2.67599771177629e-05, 2.5626337446738034e-05, 2.4625525838928297e-05, 2.3790360501152463e-05, 2.3058248189045116e-05, 2.2424499547923915e-05, 2.1909745555603877e-05, 2.1327648937585764e-05, 2.102909274981357e-05, 2.086916174448561e-05, 2.080703961837571e-05 ] } } }, "post_attention_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "bf290f184e14eee43bbcd88e48df7e026abc6363843b7efb2c6be3c7d0a34786", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.002919268561527133, 0.0065401275642216206, 0.010618876665830612, 0.015120964497327805, 0.0030634324066340923, 0.007598300464451313, 0.012669830583035946, 0.017725100740790367, 0.0037359180860221386, 0.009108707308769226, 0.015153576619923115, 0.021781234070658684, 0.005531937349587679, 0.0119093032553792, 0.01904802769422531, 0.02626260370016098, 0.005659298039972782, 0.013823030516505241, 0.02223494090139866, 0.0319429449737072, 0.007882392033934593, 0.01784195750951767, 0.028768405318260193, 0.040818072855472565, 0.012831038795411587, 0.023826664313673973, 0.03824474662542343, 0.053081437945365906, 0.013431218452751637, 0.03125866502523422, 0.04969283565878868, 0.06860727071762085 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.57084181182654e-05, "first_to_last_ratio": 3.4725850785839505, "imbalance_abs_log_ratio": 1.2448992960183491, "last_quartile_mean": 4.523551695001515e-06, "mean": 8.68330698722275e-06, "normalized": [ 2.2299043393712443, 2.1042761707719917, 1.976933189224246, 1.8613268098100455, 1.7187475287766918, 1.6201752163443084, 1.525403311499459, 1.435524363342176, 1.3017427682121105, 1.2100915045221226, 1.1332349539879822, 1.0535572156473645, 0.9690912612097308, 0.9035997979344527, 0.843793758654846, 0.7926704001101033, 0.7497625268960744, 0.7082004131597335, 0.6752277610326668, 0.6475699083216085, 0.6244433954092409, 0.6007951696153412, 0.5813694559888941, 0.5649745160906161, 0.5550358385641657, 0.5437760679691019, 0.5335476779691889, 0.5248102179054668, 0.5123899215671036, 0.50517146191784, 0.49925770343476866, 0.49359537473931375 ], "peak_layer": 1, "peak_normalized": 2.2299043393712443, "population_cv": 0.5276847791721199, "rest_mean": 9.350114733361235e-06, "spike_contrast": 0.543581047186555, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.082545158074936e-06, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 1.9362943930900656e-05, 1.8272075976710767e-05, 1.716631777526345e-05, 1.61624720931286e-05, 1.4924412425898481e-05, 1.4068478776607662e-05, 1.3245545233075973e-05, 1.2465098734537605e-05, 1.1303432074782904e-05, 1.0507596016395837e-05, 9.840226994128898e-06, 9.148360732069705e-06, 8.414916919718962e-06, 7.846234439057298e-06, 7.32692024030257e-06, 6.883000423840713e-06, 6.510418188554468e-06, 6.149521595943952e-06, 5.863209935341729e-06, 5.6230483096442185e-06, 5.4222336984821595e-06, 5.2168888942105696e-06, 5.0482094593462534e-06, 4.905847163172439e-06, 4.819546575163258e-06, 4.721774530480616e-06, 4.632958280126331e-06, 4.557088232104434e-06, 4.449238986126147e-06, 4.3865588850167114e-06, 4.33520790465991e-06, 4.286040166334715e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 6.106540149630746e-05, "first_to_last_ratio": 3.3966554168443395, "imbalance_abs_log_ratio": 1.2227912465426896, "last_quartile_mean": 1.7978097275772598e-05, "mean": 3.421876306219929e-05, "normalized": [ 2.125681514851671, 2.035741091532698, 1.9424547124055425, 1.8443747880483625, 1.7129915891324252, 1.6251025199478857, 1.537504861060443, 1.452620900762345, 1.3158482592376062, 1.2267970572702998, 1.1490758791488886, 1.0695328816728442, 0.9835696082276737, 0.9163259113865182, 0.85522266858747, 0.8029399296205194, 0.7578942872402048, 0.7156133216460457, 0.6817566226795772, 0.6538643195822892, 0.6299348137903584, 0.6059197773220268, 0.5863493060398601, 0.5697864420541977, 0.5595343492561913, 0.5483692204744319, 0.5380517969111698, 0.5293376615585488, 0.5166827361721865, 0.509484072574387, 0.5036056858005552, 0.49803141400477713 ], "peak_layer": 1, "peak_normalized": 2.125681514851671, "population_cv": 0.512158847941706, "rest_mean": 3.6814922000134914e-05, "spike_contrast": 0.5486771042805106, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.019950479734689e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 7.273819210240617e-05, 6.966054206714034e-05, 6.646839756285772e-05, 6.311222387012094e-05, 5.861645331606269e-05, 5.560899808187969e-05, 5.261151454760693e-05, 4.9706890422385186e-05, 4.502669980865903e-05, 4.1979477828135714e-05, 3.931995524908416e-05, 3.659809226519428e-05, 3.365653537912294e-05, 3.1355539249489084e-05, 2.926466186181642e-05, 2.7475611204863526e-05, 2.5934205041266978e-05, 2.4487402697559446e-05, 2.332886833755765e-05, 2.2374428226612508e-05, 2.1555590137722902e-05, 2.073382529488299e-05, 2.006414797506295e-05, 1.9497387256706133e-05, 1.9146573322359473e-05, 1.8764516426017508e-05, 1.841146695369389e-05, 1.8113280020770617e-05, 1.7680244127404876e-05, 1.7433914763387293e-05, 1.7232763639185578e-05, 1.704201895336155e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.05415340978652239, "first_to_last_ratio": 3.3966554118601855, "imbalance_abs_log_ratio": 1.2227912450753184, "last_quartile_mean": 0.015943156788125634, "mean": 0.030345541599672288, "normalized": [ 2.1256815759027154, 2.0357410833262275, 1.9424548253937048, 1.8443746973080621, 1.712991596235519, 1.6251025854549026, 1.5375048897542698, 1.4526208560186746, 1.3158482418228976, 1.2267970397813441, 1.1490758214573777, 1.0695328097128658, 0.9835695720716091, 0.9163258756382576, 0.8552226910646754, 0.8029399206669349, 0.7578942982897275, 0.715613345638086, 0.6817566530269693, 0.6538643082892013, 0.6299347939093796, 0.6059197755459048, 0.5863493070405259, 0.5697864549709054, 0.5595343861542013, 0.5483692109047464, 0.5380518325125886, 0.5293376691592189, 0.5166827726726805, 0.5094840492707883, 0.5036056657399419, 0.49803139526509604 ], "peak_layer": 1, "peak_normalized": 2.1256815759027154, "population_cv": 0.51215885272173, "rest_mean": 0.03264784129957358, "spike_contrast": 0.5486771102516746, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.01791312322020531, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0645049586892128, 0.06177566573023796, 0.05894484370946884, 0.055968549102544785, 0.05198165774345398, 0.04931461811065674, 0.04665641859173775, 0.04408056661486626, 0.03993012756109238, 0.0372278206050396, 0.03486932814121246, 0.032455552369356155, 0.029846951365470886, 0.02780640497803688, 0.02595219574868679, 0.02436564676463604, 0.022998712956905365, 0.021715674549341202, 0.02068827487528324, 0.019841866567730904, 0.019115712493658066, 0.01838696375489235, 0.017793087288737297, 0.017290478572249413, 0.01697937399148941, 0.01664056070148945, 0.016327474266290665, 0.016063038259744644, 0.015679018571972847, 0.01546056941151619, 0.015282186679542065, 0.015113032422959805 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 5.484899475050042e-06, "first_to_last_ratio": 4.317013540640096, "imbalance_abs_log_ratio": 1.4625638531540355, "last_quartile_mean": 1.2705309870852943e-06, "mean": 2.7753340354763623e-06, "normalized": [ 2.6423318625200567, 2.4162339509651107, 2.211337495963179, 2.033398824132887, 1.8430633472212699, 1.6940067504930827, 1.5445742745339275, 1.4254731551929667, 1.2827132729228703, 1.1755637695304801, 1.092360755278899, 1.0016920978710038, 0.9227083034090567, 0.8576286815024052, 0.7936258208713879, 0.7371254703657203, 0.6937452548738572, 0.6483007348957919, 0.6136555742307378, 0.5846345106553689, 0.5617683465972004, 0.5391257192612848, 0.5197876012499301, 0.5027932393500675, 0.49022455226643835, 0.4787926786546725, 0.46965294739774904, 0.460864066796148, 0.45018919494858056, 0.44356620781452427, 0.43718650171280415, 0.43187503652054116 ], "peak_layer": 1, "peak_normalized": 2.6423318625200567, "population_cv": 0.6359510157609364, "rest_mean": 3.020622224775488e-06, "spike_contrast": 0.48029104777209103, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.4507778132610838e-06, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 7.333353551075561e-06, 6.705856321786996e-06, 6.137200216471683e-06, 5.643360964313615e-06, 5.115116437082179e-06, 4.701434590970166e-06, 4.28670955443522e-06, 3.956164164264919e-06, 3.5599578041001223e-06, 3.2625821404508315e-06, 3.031665983144194e-06, 2.7800301722891163e-06, 2.560823759267805e-06, 2.380206069574342e-06, 2.2025767520972295e-06, 2.0457694063225063e-06, 1.9253748178016394e-06, 1.7992510947806295e-06, 1.7030992012223578e-06, 1.622556055735913e-06, 1.559094812364492e-06, 1.496253958066518e-06, 1.4425842209675466e-06, 1.3954191899756552e-06, 1.3605368849312072e-06, 1.3288096170072095e-06, 1.3034438097747625e-06, 1.2790517303074012e-06, 1.249425395144499e-06, 1.2310443935348303e-06, 1.2133385780543904e-06, 1.1985974879280548e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 5.00092546644737e-05, "first_to_last_ratio": 2.8610718587446584, "imbalance_abs_log_ratio": 1.0511963304057041, "last_quartile_mean": 1.7479202597314725e-05, "mean": 3.0017626613698667e-05, "normalized": [ 1.9665203436980145, 1.8844342899401736, 1.80165076038572, 1.7146572887379672, 1.6027118780052416, 1.5258449550389437, 1.4518445303422347, 1.3803062982231078, 1.2718602647538741, 1.1969127030718432, 1.1317702815894917, 1.0650354822632313, 0.9973554850676779, 0.9413624795803887, 0.8896480124159175, 0.8445590199428843, 0.809061501322106, 0.7713306058151453, 0.7401513167984001, 0.7139298020909397, 0.6922990839843121, 0.6685043123895319, 0.6485424460151992, 0.631323216874115, 0.6206922556356658, 0.6084815208074952, 0.5970366156159529, 0.5872145081510212, 0.5725964819678343, 0.5642525868906152, 0.5573685431962108, 0.5507411293887441 ], "peak_layer": 1, "peak_normalized": 1.9665203436980145, "population_cv": 0.4378481821603461, "rest_mean": 3.195058242353197e-05, "spike_contrast": 0.6128109021943274, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.9579665240598844e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 5.903027340536937e-05, 5.6566244893474504e-05, 5.4081279813544825e-05, 5.14699422637932e-05, 4.810960672330111e-05, 4.580224413075484e-05, 4.358092701295391e-05, 4.1433519072597846e-05, 3.8178226532181725e-05, 3.592847861000337e-05, 3.397305772523396e-05, 3.196983743691817e-05, 2.9938244551885873e-05, 2.825746742018964e-05, 2.670512185432017e-05, 2.5351657313876785e-05, 2.4286106054205447e-05, 2.3153514121077023e-05, 2.2217585865291767e-05, 2.1430478227557614e-05, 2.0781175408046693e-05, 2.006691283895634e-05, 1.9467704987619072e-05, 1.895082459668629e-05, 1.8631708371685818e-05, 1.8265171092934906e-05, 1.7921622202266008e-05, 1.7626785847824067e-05, 1.7187987396027893e-05, 1.693752346909605e-05, 1.673088081588503e-05, 1.6531941582798027e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 3.890059224431752e-05, "first_to_last_ratio": 2.166537206678912, "imbalance_abs_log_ratio": 0.7731301356847547, "last_quartile_mean": 1.7955192333829473e-05, "mean": 2.5870607089473197e-05, "normalized": [ 1.7831451783784935, 1.6977459463469209, 1.6200768511580206, 1.5334592693492033, 1.446421368117004, 1.3744004997462946, 1.3169700168196983, 1.2570593821131775, 1.1875487586493927, 1.1204242107881341, 1.071956417312422, 1.0193758496059768, 0.9782769348268451, 0.9375232472756838, 0.9065756301529028, 0.8773601959197916, 0.8648597359592212, 0.8401947622081907, 0.814617151100937, 0.7925804041893227, 0.77941283303955, 0.7586878792831788, 0.7416534116249156, 0.7273676136251142, 0.7223613266422326, 0.7135679461862469, 0.7058096849258172, 0.6989156870230683, 0.6866714405082665, 0.6814500720405863, 0.6750012133598903, 0.6685290817235002 ], "peak_layer": 1, "peak_normalized": 1.7831451783784935, "population_cv": 0.3297373433822509, "rest_mean": 2.7087979105999693e-05, "spike_contrast": 0.7123749662061754, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.9296798200230114e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 4.61310482933186e-05, 4.392171831568703e-05, 4.19123716710601e-05, 3.967152224504389e-05, 3.741979890037328e-05, 3.5556575312511995e-05, 3.407081385375932e-05, 3.2520889362785965e-05, 3.0722607334610075e-05, 2.8986054530832916e-05, 2.7732163289329037e-05, 2.6371872081654146e-05, 2.5308618205599487e-05, 2.4254295567516237e-05, 2.3453661924577318e-05, 2.2697840904584154e-05, 2.2374446416506544e-05, 2.1736348571721464e-05, 2.107464024447836e-05, 2.0504536223597825e-05, 2.016388316405937e-05, 1.962771602848079e-05, 1.9187024008715525e-05, 1.8817441741703078e-05, 1.8687926058191806e-05, 1.8460435967426747e-05, 1.825972503866069e-05, 1.808137312764302e-05, 1.776460703695193e-05, 1.762952706485521e-05, 1.7462691175751388e-05, 1.7295253201154992e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00011785902643168811, "first_to_last_ratio": 5.2723364637525565, "imbalance_abs_log_ratio": 1.6624736160576234, "last_quartile_mean": 2.235423085039656e-05, "mean": 5.981010565392353e-05, "normalized": [ 2.341513132793844, 2.2440750819699944, 2.142476081227823, 2.0324729445905114, 1.8907305822050442, 1.7814901971753774, 1.70505397052119, 1.6266178018526998, 1.4513520805328384, 1.3486545714944895, 1.25655955310435, 1.1590271011245943, 1.0440203382713282, 0.9503717152908809, 0.8638113904441559, 0.7921778834221871, 0.7124380420896897, 0.6459278658421956, 0.5962877600798272, 0.5569469907381962, 0.5166814306914113, 0.4757859010752184, 0.4474223682075307, 0.42807794732183224, 0.41212661540589873, 0.3977379814905876, 0.3854116965392957, 0.3748299777949146, 0.36298713381472003, 0.3565353744030076, 0.3515636207422689, 0.3488348677420969 ], "peak_layer": 1, "peak_normalized": 2.341513132793844, "population_cv": 0.6457650964234652, "rest_mean": 6.583521119157215e-05, "spike_contrast": 0.4142849283380522, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.7274535750620998e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0001400461478624493, 0.00013421836774796247, 0.00012814172077924013, 0.00012156242155469954, 0.00011308479588478804, 0.00010655111691448838, 0.00010197945812251419, 9.728818258736283e-05, 8.68055212777108e-05, 8.066317241173238e-05, 7.515495963161811e-05, 6.932153337402269e-05, 6.244296673685312e-05, 5.684183270204812e-05, 5.1664650527527556e-05, 4.7380242904182523e-05, 4.2610994569258764e-05, 3.863301390083507e-05, 3.566403393051587e-05, 3.3311058359686285e-05, 3.090277095907368e-05, 2.845680501195602e-05, 2.6760379114421085e-05, 2.5603387257433496e-05, 2.4649336410220712e-05, 2.3788750695530325e-05, 2.305151429027319e-05, 2.2418620574171655e-05, 2.1710298824473284e-05, 2.1324418412405066e-05, 2.1027057300671004e-05, 2.0863850295427255e-05 ] } } }, "post_mlp_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "ffee6151d0503764dd222b0cc68dfba65cc60aff853e7f321600f35337958883", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.005866481456905603, 0.010142817161977291, 0.01482025533914566, 0.01956234686076641, 0.0067119961604475975, 0.011908270418643951, 0.017094247043132782, 0.02295343205332756, 0.007494525518268347, 0.013766046613454819, 0.02038806490600109, 0.02682213857769966, 0.009193498641252518, 0.016781754791736603, 0.024297412484884262, 0.03140362724661827, 0.009884056635200977, 0.018740201368927956, 0.02799132652580738, 0.03797563910484314, 0.012623735703527927, 0.02351362258195877, 0.034899674355983734, 0.047332651913166046, 0.01744839735329151, 0.029514454305171967, 0.044660378247499466, 0.0601838044822216, 0.018972838297486305, 0.0376126803457737, 0.0565216988325119, 0.07583927363157272 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.5008519199000148e-05, "first_to_last_ratio": 3.366911210627585, "imbalance_abs_log_ratio": 1.2139957693612529, "last_quartile_mean": 4.457652210021479e-06, "mean": 8.299272167278104e-06, "normalized": [ 2.256857417490706, 2.1152794126450303, 1.976744865869958, 1.8670032822761609, 1.7142943873365317, 1.61284428729019, 1.5135327831047092, 1.4107548139296946, 1.2736784355647999, 1.19029358022618, 1.1070446131205842, 1.032219511002358, 0.948055380482258, 0.88484028406668, 0.831196979445672, 0.7835724276747893, 0.7420372577590195, 0.7071991859856492, 0.678238718639715, 0.6514579423864029, 0.6287823606114806, 0.6086317246696183, 0.5913247143847147, 0.5772065844972074, 0.5691335632017953, 0.558269353765152, 0.5491862116730458, 0.5409941785864528, 0.5285468164414351, 0.5224629687441996, 0.516466078557371, 0.5118498785704394 ], "peak_layer": 1, "peak_normalized": 2.256857417490706, "population_cv": 0.5242114468317668, "rest_mean": 8.921693312947389e-06, "spike_contrast": 0.5535045654951547, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.938197980663972e-06, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 1.8730273950495757e-05, 1.7555279555381276e-05, 1.6405543647124432e-05, 1.5494768376811408e-05, 1.4227395695343148e-05, 1.3385433703660965e-05, 1.256122050108388e-05, 1.1708238162100315e-05, 1.0570603990345262e-05, 9.878570381260943e-06, 9.187664545606822e-06, 8.566670658183284e-06, 7.868169632274657e-06, 7.343530342041049e-06, 6.8983299570390955e-06, 6.5030808400479145e-06, 6.158369160402799e-06, 5.869238520972431e-06, 5.628887720376952e-06, 5.406626769399736e-06, 5.218435944698285e-06, 5.051200332673034e-06, 4.907564743916737e-06, 4.7903945414873306e-06, 4.723394340544473e-06, 4.63322930954746e-06, 4.5578458411910105e-06, 4.489857929002028e-06, 4.386553882795852e-06, 4.336062374932226e-06, 4.286292551114457e-06, 4.247981451044325e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 5.8205007462674985e-05, "first_to_last_ratio": 3.284886533563498, "imbalance_abs_log_ratio": 1.189332110567881, "last_quartile_mean": 1.7719031347951386e-05, "mean": 3.265063452317918e-05, "normalized": [ 2.145023601546889, 2.046116178200582, 1.9398041355219169, 1.8522906626530224, 1.7094551367874427, 1.616905863157966, 1.5255793277696308, 1.4261117891382666, 1.2879069358899493, 1.2060838643982497, 1.1221569343801026, 1.0481379278736214, 0.9613520366262555, 0.8968015077026028, 0.8421366180143057, 0.7933562257498653, 0.7503570432885839, 0.714837243906681, 0.6855582118161445, 0.6585243205638621, 0.6351846609275962, 0.6146934889106834, 0.5972735915655352, 0.5828673640378894, 0.5748017373594797, 0.5639401582431183, 0.5548451094205582, 0.5465856652489405, 0.5340146592968225, 0.5278533354505021, 0.5219685037625007, 0.5174761607904339 ], "peak_layer": 1, "peak_normalized": 2.145023601546889, "population_cv": 0.5072011380661324, "rest_mean": 3.5063370281325966e-05, "spike_contrast": 0.5596113913680666, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.962186142918654e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 7.003638165770099e-05, 6.680699152639136e-05, 6.333583587547764e-05, 6.047846545698121e-05, 5.581479490501806e-05, 5.2793002396356314e-05, 4.981113306712359e-05, 4.656345481635071e-05, 4.2050978663610294e-05, 3.9379403460770845e-05, 3.663913594209589e-05, 3.422236841288395e-05, 3.138875399599783e-05, 2.928113826783374e-05, 2.7496294933371246e-05, 2.5903584173647687e-05, 2.449963358230889e-05, 2.3339889594353735e-05, 2.2383910618373193e-05, 2.1501236915355548e-05, 2.0739182218676433e-05, 2.0070132450200617e-05, 1.950136174855288e-05, 1.9030989278689958e-05, 1.87676414498128e-05, 1.8413003999739885e-05, 1.8116044884664007e-05, 1.7846368791651912e-05, 1.7435917470720597e-05, 1.7234746337635443e-05, 1.7042602848960087e-05, 1.689592500042636e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.05161678325384855, "first_to_last_ratio": 3.2848865977727097, "imbalance_abs_log_ratio": 1.189332130114739, "last_quartile_mean": 0.015713414060883224, "mean": 0.028954909328604117, "normalized": [ 2.1450237245911468, 2.046116217659094, 1.9398041664812302, 1.8522906413119131, 1.7094550632279355, 1.6169058509968128, 1.5255792788818974, 1.4261118508948372, 1.2879069434477655, 1.206083847584953, 1.1221569306992827, 1.0481379275281273, 0.9613519652186596, 0.8968015075628347, 0.8421366414011558, 0.7933562213086081, 0.7503569989748414, 0.7148372643644912, 0.6855582307408538, 0.6585243493162974, 0.6351846345398151, 0.614693479533182, 0.5972735918667607, 0.5828673969375834, 0.5748016775305409, 0.5639401482692175, 0.5548451622169414, 0.5465856833639338, 0.5340146386986252, 0.5278533199019263, 0.5219685200223859, 0.5174761249263505 ], "peak_layer": 1, "peak_normalized": 2.1450237245911468, "population_cv": 0.507201147039954, "rest_mean": 0.031094547602589482, "spike_contrast": 0.5596113785438716, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.017400862649083138, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.06210896745324135, 0.05924510955810547, 0.05616685375571251, 0.05363290756940842, 0.04949711635708809, 0.0468173623085022, 0.04417300969362259, 0.0412929393351078, 0.03729122877120972, 0.034922048449516296, 0.03249195218086243, 0.03034873865544796, 0.02783585898578167, 0.025966806337237358, 0.024383990094065666, 0.022971557453274727, 0.021726518869400024, 0.020698048174381256, 0.01985027641057968, 0.019067512825131416, 0.018391713500022888, 0.017798393964767456, 0.017294002696871758, 0.01687687262892723, 0.016643330454826355, 0.016328835859894753, 0.01606549136340618, 0.015826338902115822, 0.015462345443665981, 0.015283945016562939, 0.015113551169633865, 0.014983474276959896 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 5.35499910370163e-06, "first_to_last_ratio": 4.280329422122994, "imbalance_abs_log_ratio": 1.4540299744175837, "last_quartile_mean": 1.251071722663255e-06, "mean": 2.683941740855289e-06, "normalized": [ 2.703190107906874, 2.4622030057009887, 2.2387169403666407, 2.052519103467962, 1.857495181173395, 1.6956946287041017, 1.543681691896796, 1.4080949959710705, 1.2505201784095583, 1.154791072279902, 1.0675304639467325, 0.9762128756212061, 0.8974710865803879, 0.8318006025975303, 0.7746066644618449, 0.7211640498251165, 0.6781926692188598, 0.6399504079116765, 0.6084710506699328, 0.581158338700675, 0.5581318067356622, 0.5402181590129327, 0.5217735093776118, 0.5073536475319059, 0.4964690402033343, 0.48529694803264734, 0.47725139647875997, 0.4696126527079396, 0.4585674232309517, 0.45305527095834874, 0.4466379671326055, 0.4421670631860494 ], "peak_layer": 1, "peak_normalized": 2.703190107906874, "population_cv": 0.645765617676911, "rest_mean": 2.9201339657447534e-06, "spike_contrast": 0.48234216065938457, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.4085037264521817e-06, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 7.255204764078371e-06, 6.608409421460237e-06, 6.008585842209868e-06, 5.508841695700539e-06, 4.985408850188833e-06, 4.551145593723049e-06, 4.143151727475924e-06, 3.779244934776216e-06, 3.3563233046152163e-06, 3.099391960859066e-06, 2.8651895718212472e-06, 2.6200984848401276e-06, 2.408760110483854e-06, 2.232504357380094e-06, 2.078999159493833e-06, 1.9355622953298735e-06, 1.8202296132585616e-06, 1.7175896118715173e-06, 1.6331008509951062e-06, 1.5597951232848573e-06, 1.497993252996821e-06, 1.44991406614281e-06, 1.400409701091121e-06, 1.3617076319860644e-06, 1.3324939800440916e-06, 1.3025087355345022e-06, 1.2809149438908207e-06, 1.2604130006366177e-06, 1.2307682482060045e-06, 1.2159739526396152e-06, 1.1987502830379526e-06, 1.186750637316436e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 4.7860596623650054e-05, "first_to_last_ratio": 2.779352348042467, "imbalance_abs_log_ratio": 1.022217932231161, "last_quartile_mean": 1.7220053678101976e-05, "mean": 2.88126707914671e-05, "normalized": [ 1.9752636229242324, 1.8880994931589683, 1.7945585688576082, 1.7155011651574092, 1.5967516750805444, 1.5186230631712434, 1.44161945600051, 1.358346630504372, 1.2494545051794677, 1.1811062374956933, 1.1110457151250825, 1.0496228553992981, 0.981726104768848, 0.9275032753698961, 0.8805136203767746, 0.8381981012329272, 0.8039749772857018, 0.7714588681042523, 0.7440936013254396, 0.7183075709964174, 0.6966458004631945, 0.6758729225629136, 0.6578613567984469, 0.6426061157750497, 0.6340386554461201, 0.6220565974014273, 0.6118747994647586, 0.6024343556608059, 0.5879125773227442, 0.5807482794696509, 0.5737976188800432, 0.5683818132401579 ], "peak_layer": 1, "peak_normalized": 1.9752636229242324, "population_cv": 0.4317074018350235, "rest_mean": 3.0619305444243194e-05, "spike_contrast": 0.62238001123762, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.9056843666476197e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 5.6912620493676513e-05, 5.440118911792524e-05, 5.170602526050061e-05, 4.942817031405866e-05, 4.600668034981936e-05, 4.375558637548238e-05, 4.1536906792316586e-05, 3.9137594285421073e-05, 3.6000121326651424e-05, 3.4030825190711766e-05, 3.201219442416914e-05, 3.0242437787819654e-05, 2.8286151064094156e-05, 2.672384653124027e-05, 2.5369949071318842e-05, 2.4150725948857144e-05, 2.3164666345110163e-05, 2.222779039584566e-05, 2.1439323973027058e-05, 2.0696359570138156e-05, 2.00722261070041e-05, 1.9473704014671966e-05, 1.8954742699861526e-05, 1.85151984624099e-05, 1.8268347048433498e-05, 1.7923111954587512e-05, 1.762974716257304e-05, 1.7357742763124406e-05, 1.6939331544563174e-05, 1.6732908989069983e-05, 1.653264189371839e-05, 1.6376598068745807e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 3.7207921195658855e-05, "first_to_last_ratio": 2.0918054721335677, "imbalance_abs_log_ratio": 0.7380275553273271, "last_quartile_mean": 1.7787467186280992e-05, "mean": 2.50158799417477e-05, "normalized": [ 1.7669834832946238, 1.6851855745889466, 1.5913246601136481, 1.5163874050648884, 1.4314556947301431, 1.3660671618942286, 1.3038144666622677, 1.2377581340639199, 1.1604947777085126, 1.1096820799312197, 1.0568639468917405, 1.0113831764859333, 0.9697099699695068, 0.9379529791327526, 0.9077702331627978, 0.8815756711399201, 0.8691427723616427, 0.8426692090660671, 0.8202682499619266, 0.8014514792168752, 0.7847031862420706, 0.7670164542718427, 0.7519889963555914, 0.7399739814225563, 0.7380541298189486, 0.7300354427325324, 0.7228094770104511, 0.7167792102532495, 0.7048467977298535, 0.6982014484170743, 0.6912475034823692, 0.6864022468218989 ], "peak_layer": 1, "peak_normalized": 1.7669834832946238, "population_cv": 0.3174743127828502, "rest_mean": 2.6144617987921703e-05, "spike_contrast": 0.7236936680868339, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.8920694492408076e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 4.420264667714946e-05, 4.21564000134822e-05, 3.9808386645745486e-05, 3.793376527028158e-05, 3.58091238013003e-05, 3.417337211431004e-05, 3.26160661643371e-05, 3.096360887866467e-05, 2.9030798032181337e-05, 2.7759673685068265e-05, 2.6438381610205397e-05, 2.5300640118075535e-05, 2.4258148187072948e-05, 2.3463719116989523e-05, 2.2708671167492867e-05, 2.2053391148801893e-05, 2.1742371245636605e-05, 2.1080111764604226e-05, 2.0519732061075047e-05, 2.004901398322545e-05, 1.9630040696938522e-05, 1.918759153340943e-05, 1.881166645034682e-05, 1.851110027928371e-05, 1.846307350206189e-05, 1.826247898861766e-05, 1.8081715097650886e-05, 1.793086266843602e-05, 1.763236286933534e-05, 1.746612360875588e-05, 1.729216455714777e-05, 1.7170956198242493e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00011250466286583105, "first_to_last_ratio": 5.136757236447842, "imbalance_abs_log_ratio": 1.6364219925229777, "last_quartile_mean": 2.1901884338149102e-05, "mean": 5.649989418543555e-05, "normalized": [ 2.3823389400332498, 2.2792913554127336, 2.1636479699592535, 2.0682892196556057, 1.89270947379848, 1.8094109355775225, 1.7253047001597102, 1.6089010079061294, 1.4276863120740728, 1.3301882974473787, 1.227270134970172, 1.1372581289133685, 1.0077700511385481, 0.9144159274954295, 0.8389896038222593, 0.761607782391717, 0.6842191357138161, 0.6319663297187056, 0.5899873238938539, 0.550774281755648, 0.5037238917813971, 0.4736966989455727, 0.453500437871209, 0.43589448694769534, 0.4210071093140732, 0.4081032387076438, 0.39686588746098733, 0.38782343274226294, 0.37750089000360326, 0.37216747430584624, 0.36939626290542954, 0.36829327717662536 ], "peak_layer": 1, "peak_normalized": 2.3823389400332498, "population_cv": 0.6517280428580817, "rest_mean": 6.217536584127056e-05, "spike_contrast": 0.4157972678427941, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.5852347243926487e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0001346018980257213, 0.00012877972039859742, 0.00012224588135723025, 0.00011685812205541879, 0.00010693788499338552, 0.00010223152639809996, 9.747953299665824e-05, 9.090273670153692e-05, 8.066412556217983e-05, 7.515549805248156e-05, 6.934063276275992e-05, 6.425496394513175e-05, 5.6938901252578944e-05, 5.1664403144968674e-05, 4.740282383863814e-05, 4.3030759115936235e-05, 3.8658308767480776e-05, 3.570603075786494e-05, 3.333422137075104e-05, 3.111868863925338e-05, 2.8460346584324725e-05, 2.676381336641498e-05, 2.5622726752771996e-05, 2.4627992388559505e-05, 2.3786857127561234e-05, 2.3057789803715423e-05, 2.242288064735476e-05, 2.1911982912570238e-05, 2.132876034011133e-05, 2.1027422917541116e-05, 2.08708497666521e-05, 2.080853118968662e-05 ] } } }, "pre_attention_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "7fe691f51fd0a4122bf92a11e80f2ae9c1ff5b2e6ee12dad9ccd027c578f75ec", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.028571506962180138, 0.01502692885696888, 0.015751885250210762, 0.016715344041585922, 0.01783851347863674, 0.012985615991055965, 0.014028912410140038, 0.015184907242655754, 0.016659293323755264, 0.013609340414404869, 0.014725575223565102, 0.016002535820007324, 0.017315881326794624, 0.014967667870223522, 0.016154561191797256, 0.01737469807267189, 0.018479224294424057, 0.01647813245654106, 0.017671043053269386, 0.01892261393368244, 0.020303834229707718, 0.01861351728439331, 0.01984807848930359, 0.021230308338999748, 0.022779423743486404, 0.02138628251850605, 0.02279111184179783, 0.024287279695272446, 0.02611272968351841, 0.024725768715143204, 0.026407435536384583, 0.028016721829771996 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 4.961159703498197e-06, "first_to_last_ratio": 10.14195644845838, "imbalance_abs_log_ratio": 2.3166809231867567, "last_quartile_mean": 4.891718603516892e-07, "mean": 2.1833543613070105e-06, "normalized": [ 2.1149003512951285, 3.2694277026187555, 2.6224151681895895, 2.367649688300867, 1.872384189095874, 2.4215723461348544, 1.997727875442672, 1.5120426998182623, 1.407451434597839, 1.4959940566742542, 1.146826747734668, 1.0540980256857684, 0.9488948361068527, 0.9447470595114825, 0.8146834014726949, 0.6963182667169648, 0.5482907339349459, 0.5718154556650352, 0.5280062120840487, 0.46712124843842245, 0.37653152797943845, 0.3855401511265584, 0.3587208946748597, 0.2844717467264502, 0.29151483884188434, 0.260094980728135, 0.263913337696718, 0.20680766564782654, 0.1887856679838017, 0.2117583846844945, 0.19386962063621954, 0.1756236837546329 ], "peak_layer": 2, "peak_normalized": 3.2694277026187555, "population_cv": 0.8462629641613505, "rest_mean": 2.4504692364475717e-06, "spike_contrast": 0.3023641450084502, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 7.409340355479799e-07, "top_five_layers": [ 2, 3, 6, 4, 1 ], "top_five_spike_overlap": 0 }, "values": [ 4.617576905729948e-06, 7.13831923349062e-06, 5.725661594624398e-06, 5.1694182729988825e-06, 4.0880781853047665e-06, 5.2871505431539845e-06, 4.361747869552346e-06, 3.30132502313063e-06, 3.0729652280570008e-06, 3.2662851481291e-06, 2.5039291813300224e-06, 2.3014695216261316e-06, 2.071773678835598e-06, 2.0627176127163693e-06, 1.7787425576898386e-06, 1.5203095244942233e-06, 1.197112965201086e-06, 1.2484757689890102e-06, 1.1528246659509023e-06, 1.0198912150372053e-06, 8.221017537834996e-07, 8.417707704211352e-07, 7.832148298803077e-07, 6.211026288838184e-07, 6.364801947711385e-07, 5.678795105268364e-07, 5.762163368672191e-07, 4.515344187439041e-07, 4.121860115446907e-07, 4.6234359274421877e-07, 4.2328608174102555e-07, 3.834487358744809e-07 ] }, "element_rms": { "statistics": { "first_quartile_mean": 8.940424550019088e-06, "first_to_last_ratio": 7.362808193451028, "imbalance_abs_log_ratio": 1.9964414079930852, "last_quartile_mean": 1.214268294802423e-06, "mean": 4.16189458363192e-06, "normalized": [ 1.8755411782360838, 2.990879153140948, 2.507234682429089, 2.1623895678054548, 1.7817542090336904, 2.3421906478647685, 1.984613814607898, 1.5406942028635409, 1.3930431107090302, 1.467345705614427, 1.208878006326314, 1.0663129583432038, 0.9225426136071778, 0.9347168647367088, 0.8446162613986855, 0.7123769822844362, 0.5995323166764556, 0.6272780709200748, 0.55800311830824, 0.5090119191108281, 0.4207840547535497, 0.45050976377329044, 0.4129238940687079, 0.3527585690707129, 0.3423379312955225, 0.33392765114492584, 0.32776335752361546, 0.2883552290395252, 0.26514548144992983, 0.28609378158280374, 0.25631521585884554, 0.23412968642151585 ], "peak_layer": 2, "peak_normalized": 2.990879153140948, "population_cv": 0.7649777321045976, "rest_mean": 4.627515910147782e-06, "spike_contrast": 0.3560310655730732, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.6475394204462646e-06, "top_five_layers": [ 2, 3, 6, 4, 7 ], "top_five_spike_overlap": 0 }, "values": [ 7.805804671079386e-06, 1.2447723747754935e-05, 1.0434846444695722e-05, 8.99963742995169e-06, 7.415473191940691e-06, 9.747950571181718e-06, 8.259753485617694e-06, 6.412206857930869e-06, 5.797698577225674e-06, 6.106938144512242e-06, 5.0312228268012404e-06, 4.437882125785109e-06, 3.839525106741348e-06, 3.890193056577118e-06, 3.515203843562631e-06, 2.9648379040736472e-06, 2.4951903014880372e-06, 2.610665205793339e-06, 2.3223501557367854e-06, 2.1184539491514442e-06, 1.7512588783574756e-06, 1.8749741457213531e-06, 1.718545718176756e-06, 1.4681439779451466e-06, 1.4247743820305914e-06, 1.3897716826249962e-06, 1.3641165423905477e-06, 1.2001040659015416e-06, 1.1035075431209407e-06, 1.1906921599802445e-06, 1.0667569085853756e-06, 9.744230737851467e-07 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.00792845810065046, "first_to_last_ratio": 7.362808269910068, "imbalance_abs_log_ratio": 1.9964414183775794, "last_quartile_mean": 0.001076825283234939, "mean": 0.003690809797262773, "normalized": [ 1.8755411129192268, 2.990879196958467, 2.5072346853780427, 2.162389622214153, 1.7817542918869094, 2.3421908242130387, 1.9846136654580344, 1.5406942156700558, 1.3930430879772915, 1.4673456922088532, 1.2088779947214396, 1.0663129426719855, 0.9225426097974535, 0.9347168545387744, 0.8446162145203677, 0.7123769861164736, 0.5995323323039541, 0.6272780785788562, 0.5580030712234716, 0.5090119005316229, 0.4207840635329223, 0.45050976081671684, 0.4129238737745274, 0.35275859035228563, 0.3423379200763214, 0.3339276703286944, 0.3277633648171629, 0.2883552364940563, 0.2651454743312869, 0.2860937736207281, 0.25631521203611624, 0.2341296799307093 ], "peak_layer": 2, "peak_normalized": 2.990879196958467, "population_cv": 0.7649777429861212, "rest_mean": 0.004103727453319287, "spike_contrast": 0.3560310647277095, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0014610544545575976, "top_five_layers": [ 2, 3, 6, 4, 7 ], "top_five_spike_overlap": 0 }, "values": [ 0.006922265514731407, 0.011038766242563725, 0.009253726340830326, 0.007980968803167343, 0.006576116196811199, 0.008644580841064453, 0.007324831560254097, 0.005686409305781126, 0.005141457077115774, 0.005415693856775761, 0.004461738746613264, 0.003935558255761862, 0.0034049293026328087, 0.0034498621243983507, 0.0031173177994787693, 0.002629247959703207, 0.002212759805843234, 0.00231516407802701, 0.002059483202174306, 0.001878666109405458, 0.0015530339442193508, 0.001662745838984847, 0.0015240234788507223, 0.0013019648613408208, 0.0012635041493922472, 0.0012324635172262788, 0.0012097122380509973, 0.0010642643319442868, 0.0009786015143617988, 0.001055917702615261, 0.0009460106957703829, 0.000864128116518259 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 3.5133993492308946e-06, "first_to_last_ratio": 6.5659297300766655, "imbalance_abs_log_ratio": 1.881894116932073, "last_quartile_mean": 5.350954843663658e-07, "mean": 1.7443735549349526e-06, "normalized": [ 1.589774202228725, 2.5970667163618275, 2.31809881449688, 2.0431360816541173, 1.739210008196666, 2.257659305838541, 2.011325443479608, 1.5567888778424819, 1.467448725425628, 1.5902279397678998, 1.2912623477212033, 1.1632146422568665, 1.0012751149296444, 1.0140542592766522, 0.8991815616774139, 0.774122319459461, 0.6409300827695702, 0.6678856249973933, 0.6105498714134762, 0.5542747505087504, 0.4535313629747497, 0.4812253529456957, 0.4453558428285356, 0.3783599505586694, 0.3693486135392654, 0.34798394288991763, 0.34559725866794794, 0.2988109680432934, 0.27684960189841046, 0.29841376846953566, 0.272740676748336, 0.2442959701328381 ], "peak_layer": 2, "peak_normalized": 2.5970667163618275, "population_cv": 0.6989714208275406, "rest_mean": 1.9299347726515677e-06, "spike_contrast": 0.38464666774479367, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 7.423429792652314e-07, "top_five_layers": [ 2, 3, 6, 4, 7 ], "top_five_spike_overlap": 0 }, "values": [ 2.773160076685599e-06, 4.530254500423325e-06, 4.043630269734422e-06, 3.5639925499708625e-06, 3.033831944776466e-06, 3.938201189157553e-06, 3.5085029139736434e-06, 2.7156213491252856e-06, 2.559778749855468e-06, 2.773951564449817e-06, 2.2524438918480882e-06, 2.0290808606659994e-06, 1.746597831697727e-06, 1.7688894331513438e-06, 1.5685085372751928e-06, 1.3503585023499909e-06, 1.1180214869455085e-06, 1.1650420219666557e-06, 1.0650270496626035e-06, 9.66862216955633e-07, 7.911281159067585e-07, 8.394367796427105e-07, 7.768669547658646e-07, 6.600010920010391e-07, 6.442819540097844e-07, 6.070139875191671e-07, 6.028507186783827e-07, 5.212379505792342e-07, 4.829291242458567e-07, 5.205450861467398e-07, 4.7576162387485965e-07, 4.261434298769018e-07 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 4.804085307341666e-06, "first_to_last_ratio": 6.304609762674019, "imbalance_abs_log_ratio": 1.8412810741903929, "last_quartile_mean": 7.619956647886283e-07, "mean": 2.3993721356418973e-06, "normalized": [ 1.4988203443996868, 2.5234715181763763, 2.2553217408275312, 1.9958915618525483, 1.7576503840860649, 2.319660501024385, 2.0260662582488056, 1.6409258036937564, 1.4769401358690522, 1.5639642180334559, 1.318106777079787, 1.1580760773918135, 0.96371832012835, 1.0214962965942758, 0.8943161393616675, 0.7727506438550246, 0.6438659223631747, 0.6719055424540622, 0.6043445279969172, 0.5577548422477306, 0.4603517050297865, 0.4900258115330936, 0.45428577798054076, 0.38963894059637383, 0.37546196475112736, 0.3630494912829247, 0.35422949063900455, 0.315438725345089, 0.2890783675728055, 0.3094773464831404, 0.27956031292056627, 0.2543545101810816 ], "peak_layer": 2, "peak_normalized": 2.5234715181763763, "population_cv": 0.6891965904370858, "rest_mean": 2.650882836237374e-06, "spike_contrast": 0.3927802233252255, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.041214352426323e-06, "top_five_layers": [ 2, 6, 3, 7, 4 ], "top_five_spike_overlap": 0 }, "values": [ 3.5962277706858004e-06, 6.054747245798353e-06, 5.411356141848955e-06, 4.788886599271791e-06, 4.217257355776383e-06, 5.565728770307032e-06, 4.861286925006425e-06, 3.937191650038585e-06, 3.5437290080153616e-06, 3.752532165890443e-06, 3.1626286727259867e-06, 2.7786554710473865e-06, 2.312318883923581e-06, 2.4509497507096967e-06, 2.145797225239221e-06, 1.8541163626650814e-06, 1.5448739532075706e-06, 1.6121514363476308e-06, 1.4500474208034575e-06, 1.3382614270085469e-06, 1.1045550536437077e-06, 1.1757542779378127e-06, 1.090000637304911e-06, 9.348888170279679e-07, 9.008729762172152e-07, 8.710908332432155e-07, 8.499283694618498e-07, 7.56854888095404e-07, 6.936065801710356e-07, 7.4255132176404e-07, 6.707692250529362e-07, 6.102911243033304e-07 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 2.828203619742453e-06, "first_to_last_ratio": 5.846579859482356, "imbalance_abs_log_ratio": 1.765856850870783, "last_quartile_mean": 4.837364215859452e-07, "mean": 1.4834222348270032e-06, "normalized": [ 1.2646232648739053, 2.247789759160063, 2.020880547315863, 1.842393507943919, 1.7522315558445254, 2.3420641180772317, 2.0534732489830905, 1.7288628769088255, 1.548314882558432, 1.6499586193135367, 1.4182480230650574, 1.2414487105462828, 0.9740236314282787, 1.1015223442115594, 0.9385100939682994, 0.8083610783044318, 0.6755087261126067, 0.7006206946100645, 0.6299107486392663, 0.5832268076063307, 0.47664186001684955, 0.5068198063687983, 0.4772500226016884, 0.4085558916985, 0.3939522492022734, 0.36559718120224766, 0.3572742323275962, 0.3258561263199403, 0.2984295494995466, 0.3178990099824609, 0.2887716969712665, 0.26097913433726316 ], "peak_layer": 6, "peak_normalized": 2.3420641180772317, "population_cv": 0.6540082332746273, "rest_mean": 1.6337852183891368e-06, "spike_contrast": 0.41098555430286204, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.714621235914819e-07, "top_five_layers": [ 6, 2, 7, 3, 4 ], "top_five_spike_overlap": 0 }, "values": [ 1.87597026979347e-06, 3.3344213079544716e-06, 2.9978191378177144e-06, 2.7330474949849304e-06, 2.5992992505052825e-06, 3.4742699881462613e-06, 3.0461678761639632e-06, 2.564633632573532e-06, 2.2968047233007383e-06, 2.447585302434163e-06, 2.1038606519141467e-06, 1.8415926206216682e-06, 1.4448883121076506e-06, 1.6340227375621907e-06, 1.3922067410021555e-06, 1.1991407973255264e-06, 1.002064664135105e-06, 1.0393163165645092e-06, 9.34423610488011e-07, 8.651716143504018e-07, 7.070611331982946e-07, 7.518277698181919e-07, 7.079632950990344e-07, 6.06060893915128e-07, 5.843975259267609e-07, 5.42334987585491e-07, 5.299885401655047e-07, 4.833822231375962e-07, 4.4269702925703314e-07, 4.7157845983747393e-07, 4.2837035607590224e-07, 3.8714225070179964e-07 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 1.4976229522289941e-05, "first_to_last_ratio": 6.400592033457801, "imbalance_abs_log_ratio": 1.8563904913150628, "last_quartile_mean": 2.339819417329636e-06, "mean": 7.376071508247151e-06, "normalized": [ 1.5647793422352725, 2.5744682287007357, 2.3807087714141812, 2.046233253632905, 1.770279995956188, 2.2790875337951744, 2.0129055774210696, 1.6145788731579245, 1.4481155829279344, 1.5649856288248702, 1.2953241737825878, 1.1378217116015181, 0.9671807788478195, 0.9743671476553158, 0.8907704628221471, 0.7647048808528004, 0.6263127195536261, 0.6555825424956646, 0.6004321371903539, 0.5418622203213719, 0.44846797056121285, 0.4882635266638759, 0.4317412168950189, 0.383285230653721, 0.3707675618784486, 0.36766574152703, 0.3569460829427414, 0.31124413981080545, 0.2846577996070008, 0.3121781014204342, 0.2794134285368139, 0.25486763631343645 ], "peak_layer": 2, "peak_normalized": 2.5744682287007357, "population_cv": 0.7019462950147708, "rest_mean": 8.162162531435671e-06, "spike_contrast": 0.3836213712934221, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.131179983029142e-06, "top_five_layers": [ 2, 3, 6, 4, 7 ], "top_five_spike_overlap": 0 }, "values": [ 1.154192432295531e-05, 1.8989461750607006e-05, 1.756027813826222e-05, 1.5093162801349536e-05, 1.3057711839792319e-05, 1.681071262282785e-05, 1.484733547840733e-05, 1.1909249224117957e-05, 1.068140409188345e-05, 1.1543445907591376e-05, 9.554403732181527e-06, 8.392654308408964e-06, 7.1339945861836895e-06, 7.187001756392419e-06, 6.5703866312105674e-06, 5.640517883875873e-06, 4.61972740595229e-06, 4.835623713006498e-06, 4.428830379765714e-06, 3.996814484708011e-06, 3.307931820017984e-06, 3.601466687541688e-06, 3.184554088875302e-06, 2.8271392693568487e-06, 2.7348080493538873e-06, 2.7119288006360875e-06, 2.632859832374379e-06, 2.295759031767375e-06, 2.099656285281526e-06, 2.302647999385954e-06, 2.0609734292520443e-06, 1.8799219105858356e-06 ] } } }, "pre_mlp_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "6c6fc562e3dd36d26a81f8fbe75bf6f7617d8035ebd514fbd7b2faa1e7aa180f", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.014360599219799042, 0.014914286322891712, 0.015590262599289417, 0.016720104962587357, 0.01203431747853756, 0.01319553330540657, 0.014193638227880001, 0.015358141623437405, 0.012656479142606258, 0.013699201866984367, 0.01495310477912426, 0.01618308201432228, 0.01409733947366476, 0.01521329116076231, 0.016374915838241577, 0.017517324537038803, 0.015613679774105549, 0.016780734062194824, 0.017834074795246124, 0.01904122717678547, 0.017550688236951828, 0.018688000738620758, 0.019847052171826363, 0.021104788407683372, 0.019968591630458832, 0.021149026229977608, 0.02249406836926937, 0.02368815243244171, 0.02278727851808071, 0.024227207526564598, 0.02562388963997364, 0.0269329734146595 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 7.831441280359286e-06, "first_to_last_ratio": 3.949798443260095, "imbalance_abs_log_ratio": 1.373664550588347, "last_quartile_mean": 1.982744535666825e-06, "mean": 4.823673624088087e-06, "normalized": [ 1.646132756330255, 1.5986846162740493, 1.5570334831325985, 1.4112719107036644, 1.9597644907314193, 1.739595316367215, 1.586190473612319, 1.4896705111709025, 1.6742871420623147, 1.4631400905900616, 1.3644640801393357, 1.1849574810055368, 1.2519074382611204, 1.1640272230786968, 1.006304156490551, 0.9127806011294972, 0.9624471043561035, 0.8599074596436582, 0.7423282252089072, 0.6792100847159988, 0.7315895600277779, 0.6384373973703829, 0.5798636442076341, 0.5076487157583509, 0.5180109917100456, 0.4991175722294294, 0.4463413444888274, 0.40238605877214556, 0.38468977858519854, 0.36602261687556353, 0.35225030635593113, 0.31953736861450793 ], "peak_layer": 5, "peak_normalized": 1.9597644907314193, "population_cv": 0.4957051427675951, "rest_mean": 5.185350830658906e-06, "spike_contrast": 0.5536012513623684, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.870616708605667e-06, "top_five_layers": [ 5, 6, 9, 1, 2 ], "top_five_spike_overlap": 0 }, "values": [ 7.940407158457674e-06, 7.711532816756517e-06, 7.510621344408719e-06, 6.807515092077665e-06, 9.45326428336557e-06, 8.391240044147708e-06, 7.651265150343534e-06, 7.185684353316901e-06, 8.076214726315811e-06, 7.057710263325134e-06, 6.5817293943837285e-06, 5.715848146792268e-06, 6.038792889739852e-06, 5.61488741368521e-06, 4.854082817473682e-06, 4.402955710247625e-06, 4.6425307118624914e-06, 4.147912932239706e-06, 3.580749080356327e-06, 3.2762877708591986e-06, 3.5289492643642006e-06, 3.079613634326961e-06, 2.7970729661319638e-06, 2.448731720505748e-06, 2.49871595769946e-06, 2.407580268481979e-06, 2.1530049707507715e-06, 1.9409790183999576e-06, 1.8556179384177085e-06, 1.7655736428423552e-06, 1.6991405118460534e-06, 1.5413439768963144e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 3.103702101725503e-05, "first_to_last_ratio": 3.8954992147916703, "imbalance_abs_log_ratio": 1.3598218392207078, "last_quartile_mean": 7.967405281306128e-06, "mean": 1.931944555622067e-05, "normalized": [ 1.6032842527864015, 1.5688395889631517, 1.533796016047905, 1.3882386982091592, 1.9400717090010198, 1.7314283899901333, 1.585633146781874, 1.5008455649726484, 1.675406616182152, 1.4714366899853428, 1.3716626344955323, 1.1913524891080052, 1.2645012175425632, 1.1767240803978434, 1.0170067664778755, 0.9256058574210715, 0.9711818883737663, 0.8683475684259824, 0.7494840741626005, 0.6894104604993242, 0.7371611971197432, 0.6427392384511877, 0.5847748939175224, 0.5118396580731392, 0.5189878230973387, 0.5013130375398106, 0.4483794446409543, 0.404418519263501, 0.386302743223401, 0.36748312424709273, 0.35304323396633686, 0.31929937663562014 ], "peak_layer": 5, "peak_normalized": 1.9400717090010198, "population_cv": 0.4898474013710311, "rest_mean": 2.0753733493890008e-05, "spike_contrast": 0.5576967968781986, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.1574290692806243e-05, "top_five_layers": [ 5, 6, 9, 1, 7 ], "top_five_spike_overlap": 0 }, "values": [ 3.0974562832852826e-05, 3.0309111025417224e-05, 2.9632088626385666e-05, 2.682000194909051e-05, 3.74811097572092e-05, 3.345023651490919e-05, 3.063355325139128e-05, 2.8995504180784337e-05, 3.236792690586299e-05, 2.8427341021597385e-05, 2.649976158863865e-05, 2.3016269551590085e-05, 2.44294624280883e-05, 2.2733656805939972e-05, 1.9648006855277345e-05, 1.7882191968965344e-05, 1.876269561762456e-05, 1.677599357208237e-05, 1.4479616766038816e-05, 1.3319027857505716e-05, 1.4241545613913331e-05, 1.2417365724104457e-05, 1.1297526725684293e-05, 9.888458407658618e-06, 1.0026556992670521e-05, 9.68508993537398e-06, 8.662442269269377e-06, 7.813141564838588e-06, 7.46315481592319e-06, 7.099570211721584e-06, 6.820599537604721e-06, 6.168686923047062e-06 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.027523940661922097, "first_to_last_ratio": 3.8954991154967806, "imbalance_abs_log_ratio": 1.3598218137310631, "last_quartile_mean": 0.007065574871376157, "mean": 0.017132677690824494, "normalized": [ 1.6032842798207116, 1.5688395364239456, 1.5337959695585364, 1.3882387052917204, 1.940071709225884, 1.7314283707937437, 1.5856331209472507, 1.5008455668619336, 1.6754066305691406, 1.4714366809574604, 1.3716626672028296, 1.1913525502963036, 1.2645012070621602, 1.176724101825654, 1.0170067499274977, 0.9256058277235238, 0.9711818600927834, 0.8683475735699788, 0.7494840504534797, 0.6894104671183356, 0.7371612027997355, 0.6427392553352459, 0.5847748722004306, 0.5118396849118217, 0.5189878410918901, 0.5013130358668625, 0.44837945479745345, 0.40441852259963446, 0.38630275368745787, 0.36748313889002426, 0.3530432360984768, 0.31929937599809327 ], "peak_layer": 5, "peak_normalized": 1.940071709225884, "population_cv": 0.4898473968405949, "rest_mean": 0.018404618560991907, "spike_contrast": 0.5576968062611827, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.01026419699192047, "top_five_layers": [ 5, 6, 9, 1, 7 ], "top_five_spike_overlap": 0 }, "values": [ 0.027468552812933922, 0.026878422126173973, 0.02627803198993206, 0.023784246295690536, 0.033238623291254044, 0.02966400422155857, 0.02716614119708538, 0.02571350336074829, 0.02870420180261135, 0.02520965039730072, 0.02350025437772274, 0.020411059260368347, 0.021664291620254517, 0.020160434767603874, 0.017424048855900764, 0.015858106315135956, 0.016638945788145065, 0.014877119101583958, 0.01284066867083311, 0.011811447329819202, 0.012629545293748379, 0.011011844500899315, 0.010018759407103062, 0.008769184350967407, 0.008891651406884193, 0.008588834665715694, 0.00768194068223238, 0.006928772199898958, 0.0066184005700051785, 0.0062959701754152775, 0.006048575975000858, 0.005470453295856714 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 2.0978330894649844e-06, "first_to_last_ratio": 3.7123595485838843, "imbalance_abs_log_ratio": 1.3116676713818918, "last_quartile_mean": 5.650942647150714e-07, "mean": 1.3341314888393185e-06, "normalized": [ 1.5241148267186329, 1.5504517836489524, 1.525800361966804, 1.340018587688921, 1.9164903896693077, 1.732090277095866, 1.5197216980945576, 1.4707803330175575, 1.6808225666055663, 1.5059009907724967, 1.3766280212942543, 1.196783029382642, 1.2615435432700803, 1.196050358398629, 1.046816748775451, 0.9373627935785216, 0.9843932320660254, 0.8775573102072941, 0.7634158116096246, 0.6842432896373823, 0.7549093975240115, 0.6636832533885216, 0.5908781007193045, 0.5110061412689536, 0.5365346292575633, 0.5196779133499698, 0.4612664477302504, 0.4092178350653177, 0.3933915708760512, 0.3701864458066216, 0.3616807560410091, 0.33658155547385904 ], "peak_layer": 5, "peak_normalized": 1.9164903896693077, "population_cv": 0.4775581328679472, "rest_mean": 1.4301389744223006e-06, "spike_contrast": 0.570357903168614, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 8.156910666912154e-07, "top_five_layers": [ 5, 6, 9, 2, 3 ], "top_five_spike_overlap": 0 }, "values": [ 2.0333695829322096e-06, 2.068506546493154e-06, 2.035618308582343e-06, 1.7877609934657812e-06, 2.5568501769157592e-06, 2.3108361801860156e-06, 2.0275085717003094e-06, 1.9622143554443028e-06, 2.242438313260209e-06, 2.009069930863916e-06, 1.8366027916272287e-06, 1.596665924807894e-06, 1.6830649656185415e-06, 1.5956884453771636e-06, 1.3965911875857273e-06, 1.2505652193794958e-06, 1.3133100082995952e-06, 1.170776840808685e-06, 1.018497073346225e-06, 9.128705187322339e-07, 1.0071483984575025e-06, 8.85440726960951e-07, 7.883090802351944e-07, 6.817493840571842e-07, 7.158077437452448e-07, 6.933186682545056e-07, 6.153900926619826e-07, 5.45950399555295e-07, 5.248360821497045e-07, 4.938773940921237e-07, 4.825296855415218e-07, 4.4904405172019324e-07 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 2.398965625616256e-05, "first_to_last_ratio": 3.369027618649736, "imbalance_abs_log_ratio": 1.2146241622684435, "last_quartile_mean": 7.120646955627308e-06, "mean": 1.5399188356468585e-05, "normalized": [ 1.5913426512839013, 1.5418831400755482, 1.5021141560062705, 1.3562195006410476, 1.8805635081013792, 1.6560198739348433, 1.512688238851242, 1.4219848670303201, 1.6092469390600324, 1.4165297374033572, 1.3242582904493114, 1.1412781561654184, 1.2305966607275267, 1.1486603349616307, 1.006300256787253, 0.9179301571399356, 0.9843528747997042, 0.8966403626677242, 0.7866772482945813, 0.7196405302926673, 0.7789613145990362, 0.6869940328779693, 0.6281068759775288, 0.5617779475196961, 0.5746692969502841, 0.5501909649183, 0.500671447520295, 0.45822743358408635, 0.4365896209302579, 0.4181501204665046, 0.398078906415543, 0.36265455356680404 ], "peak_layer": 5, "peak_normalized": 1.8805635081013792, "population_cv": 0.4466412974933768, "rest_mean": 1.640840013418778e-05, "spike_contrast": 0.6063628797090791, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.949444756784941e-06, "top_five_layers": [ 5, 6, 9, 1, 2 ], "top_five_spike_overlap": 0 }, "values": [ 2.45053852268029e-05, 2.37437488976866e-05, 2.3131338821258396e-05, 2.088467954308726e-05, 2.8959151677554473e-05, 2.5501361960778013e-05, 2.3294171114685014e-05, 2.1897412807447836e-05, 2.478109672665596e-05, 2.181340823881328e-05, 2.0392502847244032e-05, 1.7574757293914445e-05, 1.895018976938445e-05, 1.7688436855678447e-05, 1.5496207197429612e-05, 1.4135379387880675e-05, 1.5158235328271985e-05, 1.3807533832732588e-05, 1.2114191122236662e-05, 1.108188007492572e-05, 1.1995372005912941e-05, 1.0579150512057822e-05, 9.672336091171019e-06, 8.650924428366125e-06, 8.849440746416803e-06, 8.472494300804101e-06, 7.7099339250708e-06, 7.056330559862545e-06, 6.72312580718426e-06, 6.439172466343734e-06, 6.130092060629977e-06, 5.584585778706241e-06 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 1.7287268860854965e-05, "first_to_last_ratio": 2.7170527496823964, "imbalance_abs_log_ratio": 0.9995477449345839, "last_quartile_mean": 6.362507633639325e-06, "mean": 1.1818911630712137e-05, "normalized": [ 1.5531683121051287, 1.4809645584609277, 1.4249957798471882, 1.2911504637898432, 1.7654831615088196, 1.5114625239638928, 1.4072245252680236, 1.266979301315116, 1.5685685065032353, 1.3590910493816417, 1.257358469917724, 1.0915032257961577, 1.1980939699800082, 1.1009846962306815, 1.0217422520554849, 0.917572741077425, 1.0166318370670364, 0.924735094262937, 0.841915821310778, 0.7538749370193452, 0.8515117201007877, 0.7457322090867645, 0.703238735238229, 0.6393538153043691, 0.6469780758226346, 0.6196156550317132, 0.5947215623716378, 0.5572499311933502, 0.511086746910104, 0.5033639872021801, 0.45209119593165065, 0.4215551389451849 ], "peak_layer": 5, "peak_normalized": 1.7654831615088196, "population_cv": 0.3798574342343211, "rest_mean": 1.2437515848702148e-05, "spike_contrast": 0.6816834612878747, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 8.478448853566078e-06, "top_five_layers": [ 5, 9, 1, 6, 2 ], "top_five_spike_overlap": 0 }, "values": [ 1.8356759028392844e-05, 1.7503389244666323e-05, 1.6841899196151644e-05, 1.5259993233485147e-05, 2.086608947138302e-05, 1.7863842003862374e-05, 1.6631862308713607e-05, 1.4974316400184762e-05, 1.8538772565079853e-05, 1.6062977010733448e-05, 1.4860608644085005e-05, 1.2900380170322023e-05, 1.4160166756482795e-05, 1.301244083151687e-05, 1.2075881386408582e-05, 1.0844711141544394e-05, 1.2015481843263842e-05, 1.092936236091191e-05, 9.950528692570515e-06, 8.909981261240318e-06, 1.0063941772386897e-05, 8.813743079372216e-06, 8.311516467074398e-06, 7.556466243840987e-06, 7.646576705155894e-06, 7.323182671825634e-06, 7.028961590549443e-06, 6.586087692994624e-06, 6.040489097358659e-06, 5.949214482825482e-06, 5.343225893739145e-06, 4.982322934665717e-06 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 6.156662675493862e-05, "first_to_last_ratio": 4.432028161582752, "imbalance_abs_log_ratio": 1.488857303572528, "last_quartile_mean": 1.3891298635826388e-05, "mean": 3.828891439638937e-05, "normalized": [ 1.5694774864844574, 1.533915591681154, 1.4779364465001996, 1.3758197901662568, 1.9142257050168354, 1.758664969175181, 1.6344745902568742, 1.5990784946964705, 1.7383903346505918, 1.5307661661772007, 1.4323492558141884, 1.2429043804879258, 1.3172537017266024, 1.2162602046192026, 1.0552102311500136, 0.9826538220680813, 1.0107614019133901, 0.8826720328909949, 0.739013117314438, 0.7006180982562433, 0.737790003472691, 0.6192970376209981, 0.5556544994109884, 0.47239582046029627, 0.4740970917174548, 0.4544292159121539, 0.39665333426527116, 0.35034805233110755, 0.3379110425614577, 0.30972196454961737, 0.31015209244222897, 0.26910402420943297 ], "peak_layer": 5, "peak_normalized": 1.9142257050168354, "population_cv": 0.5160505486771831, "rest_mean": 4.132475101801834e-05, "spike_contrast": 0.5298373517131695, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.1895396639592945e-05, "top_five_layers": [ 5, 6, 9, 7, 8 ], "top_five_spike_overlap": 0 }, "values": [ 6.009358912706375e-05, 5.873196278116666e-05, 5.658858208335005e-05, 5.2678646170534194e-05, 7.329362415475771e-05, 6.733737245667726e-05, 6.258225766941905e-05, 6.122697959654033e-05, 6.656107871094719e-05, 5.861137469764799e-05, 5.484309804160148e-05, 4.758945942739956e-05, 5.04362142237369e-05, 4.6569282858399674e-05, 4.0402854210697114e-05, 3.76247480744496e-05, 3.870095679303631e-05, 3.379655390745029e-05, 2.8296009986661375e-05, 2.682590638869442e-05, 2.8249178285477683e-05, 2.3712211259407923e-05, 2.1275407561915927e-05, 1.8087523130816408e-05, 1.8152662960346788e-05, 1.7399601347278804e-05, 1.5187425560725387e-05, 1.3414446584647521e-05, 1.293824698223034e-05, 1.1858917787321843e-05, 1.1875386917381547e-05, 1.0303700946678873e-05 ] } } } } }, "parent_learned_round06_exact": true, "step": 100 }, { "local_matrix": null, "parent_learned": { "bits_per_byte": 3.0609520355704922, "forward": { "activation_sha256": { "attention_branch_output": "981ba7bc004b8232ce5e00b2be2c80efd41f83b0edfc6e522ef32de2929ebcb1", "mlp_branch_output": "b9781b2e1c55e42bdc32f1e48af8d06b6449190e50665e2511e36ff3b4b6359c", "post_attention_state": "3706b7098979776b3ef475b6da00314debf1299b08b4c6a8322412051a6af174", "post_mlp_state": "2674fdc2460a10ee1ce2d4e7e0c398dced26688787f8d47d6c42ecf46862e67d", "pre_attention_input": "afa8fa86c6ecedab7e0fd84b73ca7d5e942ca1b7238fcfa0893f3ce260e81e6b", "pre_mlp_input": "65621f80daccd38bb1dc68831a9ef8c381d5d3b87c541e2988c6ff3556a7910d" }, "logits_sha256": "b2134e9ad4fc330ab7158864584c7289a039b641b1626767dcba8e1a49e62a3d", "loss_nats": 2.121690273284912, "mixer_summary_sha256": "24302def7a1bb63aa3c7a0b6a721bf1efbd647790e9d6d8427706d94cca54b03" }, "loss_nats": 2.121690273284912, "loss_scale": 1.0, "mixers": [ { "branch": "attention", "entropy_mean": -0.0, "group": 1, "latest_source_mass": 1.0, "layer": 1, "max_source_mass": 1.0, "mixer_index": 0, "normalized_entropy": 1.0, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 1.0, "median": 1.0, "p05": 1.0, "p95": 1.0 } ], "sources": 1 }, { "branch": "mlp", "entropy_mean": 0.658259391784668, "group": 1, "latest_source_mass": 0.6112743616104126, "layer": 1, "max_source_mass": 0.6112743616104126, "mixer_index": 1, "normalized_entropy": 0.9496675601463258, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.3887256383895874, "median": 0.36817848682403564, "p05": 0.2958398759365082, "p95": 0.519643247127533 }, { "label": "current_group_partial", "mean": 0.6112743616104126, "median": 0.6318215131759644, "p05": 0.48035678267478943, "p95": 0.7041600942611694 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6694332361221313, "group": 1, "latest_source_mass": 0.5721787214279175, "layer": 2, "max_source_mass": 0.5721787214279175, "mixer_index": 2, "normalized_entropy": 0.9657880099596494, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.42782124876976013, "median": 0.40344932675361633, "p05": 0.3339918255805969, "p95": 0.5918412208557129 }, { "label": "current_group_partial", "mean": 0.5721787214279175, "median": 0.596550703048706, "p05": 0.4081588089466095, "p95": 0.6660082340240479 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.6890689730644226, "group": 1, "latest_source_mass": 0.4954703748226166, "layer": 2, "max_source_mass": 0.5045296549797058, "mixer_index": 3, "normalized_entropy": 0.9941163902704933, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.5045296549797058, "median": 0.50140380859375, "p05": 0.4251278042793274, "p95": 0.583143949508667 }, { "label": "current_group_partial", "mean": 0.4954703748226166, "median": 0.49859619140625, "p05": 0.41685599088668823, "p95": 0.5748721957206726 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6740086078643799, "group": 1, "latest_source_mass": 0.5477453470230103, "layer": 3, "max_source_mass": 0.5477453470230103, "mixer_index": 4, "normalized_entropy": 0.9723888760824149, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.45225459337234497, "median": 0.42421072721481323, "p05": 0.3554007411003113, "p95": 0.6163875460624695 }, { "label": "current_group_partial", "mean": 0.5477453470230103, "median": 0.575789213180542, "p05": 0.3836124539375305, "p95": 0.644599199295044 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.679557204246521, "group": 1, "latest_source_mass": 0.5538821220397949, "layer": 3, "max_source_mass": 0.5538821220397949, "mixer_index": 5, "normalized_entropy": 0.9803938085668243, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.4461178183555603, "median": 0.43929627537727356, "p05": 0.3631950318813324, "p95": 0.562574565410614 }, { "label": "current_group_partial", "mean": 0.5538821220397949, "median": 0.5607037544250488, "p05": 0.4374254047870636, "p95": 0.63680499792099 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6740646362304688, "group": 1, "latest_source_mass": 0.5581084489822388, "layer": 4, "max_source_mass": 0.5581084489822388, "mixer_index": 6, "normalized_entropy": 0.9724697079283204, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.44189155101776123, "median": 0.42476987838745117, "p05": 0.33982762694358826, "p95": 0.5899009108543396 }, { "label": "current_group_partial", "mean": 0.5581084489822388, "median": 0.5752301216125488, "p05": 0.410099059343338, "p95": 0.6601723432540894 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.6792672276496887, "group": 1, "latest_source_mass": 0.5544062852859497, "layer": 4, "max_source_mass": 0.5544062852859497, "mixer_index": 7, "normalized_entropy": 0.9799754607686005, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.4455936849117279, "median": 0.44227513670921326, "p05": 0.348201721906662, "p95": 0.5532940626144409 }, { "label": "current_group_partial", "mean": 0.5544062852859497, "median": 0.5577248334884644, "p05": 0.4467058777809143, "p95": 0.6517983078956604 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.674970269203186, "group": 2, "latest_source_mass": 0.5532590746879578, "layer": 5, "max_source_mass": 0.5532590746879578, "mixer_index": 8, "normalized_entropy": 0.9737762601269252, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.44674086570739746, "median": 0.43155425786972046, "p05": 0.3413114547729492, "p95": 0.5883275270462036 }, { "label": "completed_group_1", "mean": 0.5532590746879578, "median": 0.5684457421302795, "p05": 0.4116724729537964, "p95": 0.6586886048316956 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 1.093062162399292, "group": 2, "latest_source_mass": 0.35190510749816895, "layer": 5, "max_source_mass": 0.35190510749816895, "mixer_index": 9, "normalized_entropy": 0.99494805735739, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.3082374334335327, "median": 0.30379241704940796, "p05": 0.26716136932373047, "p95": 0.3615886867046356 }, { "label": "completed_group_1", "mean": 0.33985745906829834, "median": 0.34154799580574036, "p05": 0.2781605124473572, "p95": 0.3903803527355194 }, { "label": "current_group_partial", "mean": 0.35190510749816895, "median": 0.35370177030563354, "p05": 0.3136623501777649, "p95": 0.3873046040534973 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0853323936462402, "group": 2, "latest_source_mass": 0.37136080861091614, "layer": 6, "max_source_mass": 0.37136080861091614, "mixer_index": 10, "normalized_entropy": 0.9879121186256079, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.30180174112319946, "median": 0.28923720121383667, "p05": 0.2442891150712967, "p95": 0.3929156959056854 }, { "label": "completed_group_1", "mean": 0.3268374800682068, "median": 0.3262294828891754, "p05": 0.2930828928947449, "p95": 0.3670434355735779 }, { "label": "current_group_partial", "mean": 0.37136080861091614, "median": 0.38435930013656616, "p05": 0.2483101636171341, "p95": 0.4514022171497345 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0921990871429443, "group": 2, "latest_source_mass": 0.3605259656906128, "layer": 6, "max_source_mass": 0.3605259656906128, "mixer_index": 11, "normalized_entropy": 0.9941624524035314, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.3034760355949402, "median": 0.30043965578079224, "p05": 0.2575032711029053, "p95": 0.3658854067325592 }, { "label": "completed_group_1", "mean": 0.3359980285167694, "median": 0.33590996265411377, "p05": 0.29249265789985657, "p95": 0.3839111328125 }, { "label": "current_group_partial", "mean": 0.3605259656906128, "median": 0.3592720031738281, "p05": 0.3142351508140564, "p95": 0.40569764375686646 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.076724648475647, "group": 2, "latest_source_mass": 0.3958718776702881, "layer": 7, "max_source_mass": 0.3958718776702881, "mixer_index": 12, "normalized_entropy": 0.9800770113185262, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.29931002855300903, "median": 0.2845022678375244, "p05": 0.24265936017036438, "p95": 0.3946967124938965 }, { "label": "completed_group_1", "mean": 0.30481812357902527, "median": 0.2940869927406311, "p05": 0.26745280623435974, "p95": 0.36952510476112366 }, { "label": "current_group_partial", "mean": 0.3958718776702881, "median": 0.4253297448158264, "p05": 0.24267785251140594, "p95": 0.4795311689376831 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0856893062591553, "group": 2, "latest_source_mass": 0.3888435363769531, "layer": 7, "max_source_mass": 0.3888435363769531, "mixer_index": 13, "normalized_entropy": 0.988236994486361, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.2801036834716797, "median": 0.2685231566429138, "p05": 0.24713139235973358, "p95": 0.3503561317920685 }, { "label": "completed_group_1", "mean": 0.3310527801513672, "median": 0.330751895904541, "p05": 0.2913045883178711, "p95": 0.37040621042251587 }, { "label": "current_group_partial", "mean": 0.3888435363769531, "median": 0.3877270221710205, "p05": 0.3418702185153961, "p95": 0.44118839502334595 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0752873420715332, "group": 2, "latest_source_mass": 0.3948463201522827, "layer": 8, "max_source_mass": 0.3948463201522827, "mixer_index": 14, "normalized_entropy": 0.9787687186488199, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.29463040828704834, "median": 0.28890830278396606, "p05": 0.23444202542304993, "p95": 0.37455716729164124 }, { "label": "completed_group_1", "mean": 0.3105233311653137, "median": 0.2965858280658722, "p05": 0.2587319016456604, "p95": 0.39421546459198 }, { "label": "current_group_partial", "mean": 0.3948463201522827, "median": 0.4135509133338928, "p05": 0.24363072216510773, "p95": 0.5001816153526306 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.088913917541504, "group": 2, "latest_source_mass": 0.3843393623828888, "layer": 8, "max_source_mass": 0.3843393623828888, "mixer_index": 15, "normalized_entropy": 0.9911721621661782, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.29400017857551575, "median": 0.2848791778087616, "p05": 0.2631606459617615, "p95": 0.35448360443115234 }, { "label": "completed_group_1", "mean": 0.32166045904159546, "median": 0.32034605741500854, "p05": 0.2860022187232971, "p95": 0.3604046106338501 }, { "label": "current_group_partial", "mean": 0.3843393623828888, "median": 0.38500386476516724, "p05": 0.336969792842865, "p95": 0.43113112449645996 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.054490327835083, "group": 3, "latest_source_mass": 0.42586958408355713, "layer": 9, "max_source_mass": 0.42586958408355713, "mixer_index": 16, "normalized_entropy": 0.959838460494086, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.2850497364997864, "median": 0.2810911536216736, "p05": 0.21189084649085999, "p95": 0.37678587436676025 }, { "label": "completed_group_1", "mean": 0.2890806794166565, "median": 0.26693958044052124, "p05": 0.21384978294372559, "p95": 0.4104633927345276 }, { "label": "completed_group_2", "mean": 0.42586958408355713, "median": 0.45180952548980713, "p05": 0.2289215475320816, "p95": 0.5656958818435669 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.3700100183486938, "group": 3, "latest_source_mass": 0.28284209966659546, "layer": 9, "max_source_mass": 0.28284209966659546, "mixer_index": 17, "normalized_entropy": 0.9882533297199292, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.21610885858535767, "median": 0.21061542630195618, "p05": 0.18654318153858185, "p95": 0.26685822010040283 }, { "label": "completed_group_1", "mean": 0.22824729979038239, "median": 0.22523541748523712, "p05": 0.19356240332126617, "p95": 0.2682129740715027 }, { "label": "completed_group_2", "mean": 0.2728016972541809, "median": 0.27193963527679443, "p05": 0.21500371396541595, "p95": 0.3283920884132385 }, { "label": "current_group_partial", "mean": 0.28284209966659546, "median": 0.2946581244468689, "p05": 0.19060052931308746, "p95": 0.3525015413761139 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3620216846466064, "group": 3, "latest_source_mass": 0.2944645285606384, "layer": 10, "max_source_mass": 0.2944645285606384, "mixer_index": 18, "normalized_entropy": 0.9824909650114454, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.22791308164596558, "median": 0.22349298000335693, "p05": 0.18910717964172363, "p95": 0.2777109742164612 }, { "label": "completed_group_1", "mean": 0.2214205265045166, "median": 0.19864881038665771, "p05": 0.15984180569648743, "p95": 0.34694117307662964 }, { "label": "completed_group_2", "mean": 0.2562018632888794, "median": 0.2503240704536438, "p05": 0.20511329174041748, "p95": 0.32819920778274536 }, { "label": "current_group_partial", "mean": 0.2944645285606384, "median": 0.30793535709381104, "p05": 0.16727104783058167, "p95": 0.3644053339958191 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.3784711360931396, "group": 3, "latest_source_mass": 0.2763957977294922, "layer": 10, "max_source_mass": 0.2763957977294922, "mixer_index": 19, "normalized_entropy": 0.994356736025074, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.23347607254981995, "median": 0.23056401312351227, "p05": 0.20095914602279663, "p95": 0.2848108112812042 }, { "label": "completed_group_1", "mean": 0.23307976126670837, "median": 0.2289559543132782, "p05": 0.205013245344162, "p95": 0.2760506868362427 }, { "label": "completed_group_2", "mean": 0.2570483684539795, "median": 0.2558031678199768, "p05": 0.21930555999279022, "p95": 0.29454779624938965 }, { "label": "current_group_partial", "mean": 0.2763957977294922, "median": 0.2743210792541504, "p05": 0.2263108193874359, "p95": 0.3325941264629364 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3619427680969238, "group": 3, "latest_source_mass": 0.302674800157547, "layer": 11, "max_source_mass": 0.302674800157547, "mixer_index": 20, "normalized_entropy": 0.9824340387540098, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.24715131521224976, "median": 0.24610307812690735, "p05": 0.21518638730049133, "p95": 0.2835894525051117 }, { "label": "completed_group_1", "mean": 0.22003203630447388, "median": 0.19626209139823914, "p05": 0.16581544280052185, "p95": 0.3317301273345947 }, { "label": "completed_group_2", "mean": 0.23014184832572937, "median": 0.2278168797492981, "p05": 0.19933417439460754, "p95": 0.272026926279068 }, { "label": "current_group_partial", "mean": 0.302674800157547, "median": 0.3237387239933014, "p05": 0.16210556030273438, "p95": 0.38430726528167725 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.3804662227630615, "group": 3, "latest_source_mass": 0.28360050916671753, "layer": 11, "max_source_mass": 0.28360050916671753, "mixer_index": 21, "normalized_entropy": 0.995795886847494, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.24302780628204346, "median": 0.24201302230358124, "p05": 0.21055130660533905, "p95": 0.276764452457428 }, { "label": "completed_group_1", "mean": 0.23108476400375366, "median": 0.22807812690734863, "p05": 0.20644722878932953, "p95": 0.2614594101905823 }, { "label": "completed_group_2", "mean": 0.24228686094284058, "median": 0.2414286881685257, "p05": 0.21957996487617493, "p95": 0.2676010727882385 }, { "label": "current_group_partial", "mean": 0.28360050916671753, "median": 0.2854132354259491, "p05": 0.246682807803154, "p95": 0.3173881471157074 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3502743244171143, "group": 3, "latest_source_mass": 0.31918564438819885, "layer": 12, "max_source_mass": 0.31918564438819885, "mixer_index": 22, "normalized_entropy": 0.974017035838133, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.2455797642469406, "median": 0.2419765591621399, "p05": 0.2023906707763672, "p95": 0.29628193378448486 }, { "label": "completed_group_1", "mean": 0.21441738307476044, "median": 0.19096502661705017, "p05": 0.1489589512348175, "p95": 0.3387751877307892 }, { "label": "completed_group_2", "mean": 0.2208172082901001, "median": 0.21868902444839478, "p05": 0.19425739347934723, "p95": 0.25476256012916565 }, { "label": "current_group_partial", "mean": 0.31918564438819885, "median": 0.34166574478149414, "p05": 0.153193861246109, "p95": 0.42244112491607666 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.3820215463638306, "group": 3, "latest_source_mass": 0.27603140473365784, "layer": 12, "max_source_mass": 0.27603140473365784, "mixer_index": 23, "normalized_entropy": 0.9969178156703975, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.24522465467453003, "median": 0.24368876218795776, "p05": 0.22978872060775757, "p95": 0.2647436261177063 }, { "label": "completed_group_1", "mean": 0.22846508026123047, "median": 0.22824203968048096, "p05": 0.21080705523490906, "p95": 0.24792201817035675 }, { "label": "completed_group_2", "mean": 0.25027889013290405, "median": 0.25114861130714417, "p05": 0.2239076942205429, "p95": 0.27430635690689087 }, { "label": "current_group_partial", "mean": 0.27603140473365784, "median": 0.2767183184623718, "p05": 0.23810210824012756, "p95": 0.31143108010292053 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3491160869598389, "group": 4, "latest_source_mass": 0.3227139413356781, "layer": 13, "max_source_mass": 0.3227139413356781, "mixer_index": 24, "normalized_entropy": 0.9731815441202416, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.24498310685157776, "median": 0.24184581637382507, "p05": 0.20811651647090912, "p95": 0.2852743864059448 }, { "label": "completed_group_1", "mean": 0.21178749203681946, "median": 0.1895410120487213, "p05": 0.14949846267700195, "p95": 0.3285062313079834 }, { "label": "completed_group_2", "mean": 0.2205154001712799, "median": 0.2162817418575287, "p05": 0.18882519006729126, "p95": 0.26559314131736755 }, { "label": "completed_group_3", "mean": 0.3227139413356781, "median": 0.34892457723617554, "p05": 0.15726026892662048, "p95": 0.42967018485069275 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.6032840013504028, "group": 4, "latest_source_mass": 0.22103822231292725, "layer": 13, "max_source_mass": 0.22103822231292725, "mixer_index": 25, "normalized_entropy": 0.9961763600595251, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.20074918866157532, "median": 0.20095838606357574, "p05": 0.18337497115135193, "p95": 0.2181507796049118 }, { "label": "completed_group_1", "mean": 0.17959120869636536, "median": 0.17876555025577545, "p05": 0.1576874852180481, "p95": 0.20805387198925018 }, { "label": "completed_group_2", "mean": 0.18619459867477417, "median": 0.18535691499710083, "p05": 0.1644289195537567, "p95": 0.21231110394001007 }, { "label": "completed_group_3", "mean": 0.21242676675319672, "median": 0.2123909741640091, "p05": 0.19269250333309174, "p95": 0.23113366961479187 }, { "label": "current_group_partial", "mean": 0.22103822231292725, "median": 0.2211262434720993, "p05": 0.1813979148864746, "p95": 0.26087528467178345 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.5699318647384644, "group": 4, "latest_source_mass": 0.2694075107574463, "layer": 14, "max_source_mass": 0.2694075107574463, "mixer_index": 26, "normalized_entropy": 0.9754535124403232, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1938496232032776, "median": 0.19102387130260468, "p05": 0.16569040715694427, "p95": 0.23268038034439087 }, { "label": "completed_group_1", "mean": 0.17019829154014587, "median": 0.14846302568912506, "p05": 0.11795832216739655, "p95": 0.2854715883731842 }, { "label": "completed_group_2", "mean": 0.1625727415084839, "median": 0.15400215983390808, "p05": 0.1330171525478363, "p95": 0.22402943670749664 }, { "label": "completed_group_3", "mean": 0.20397180318832397, "median": 0.2051275074481964, "p05": 0.14910343289375305, "p95": 0.25531816482543945 }, { "label": "current_group_partial", "mean": 0.2694075107574463, "median": 0.2924001216888428, "p05": 0.12448020279407501, "p95": 0.3373839557170868 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.6068798303604126, "group": 4, "latest_source_mass": 0.21404394507408142, "layer": 14, "max_source_mass": 0.21404394507408142, "mixer_index": 27, "normalized_entropy": 0.9984105742421472, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.19717517495155334, "median": 0.19712674617767334, "p05": 0.18285326659679413, "p95": 0.21078822016716003 }, { "label": "completed_group_1", "mean": 0.1942940056324005, "median": 0.19466334581375122, "p05": 0.18043261766433716, "p95": 0.20777744054794312 }, { "label": "completed_group_2", "mean": 0.19104912877082825, "median": 0.18969203531742096, "p05": 0.17138653993606567, "p95": 0.2142438441514969 }, { "label": "completed_group_3", "mean": 0.20343773066997528, "median": 0.2029801905155182, "p05": 0.18992920219898224, "p95": 0.2184751033782959 }, { "label": "current_group_partial", "mean": 0.21404394507408142, "median": 0.21432773768901825, "p05": 0.1852298229932785, "p95": 0.24286989867687225 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.5761134624481201, "group": 4, "latest_source_mass": 0.26560431718826294, "layer": 15, "max_source_mass": 0.26560431718826294, "mixer_index": 28, "normalized_entropy": 0.979294355048726, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1940101534128189, "median": 0.19362978637218475, "p05": 0.1631820648908615, "p95": 0.2295525223016739 }, { "label": "completed_group_1", "mean": 0.18250693380832672, "median": 0.16429191827774048, "p05": 0.13507941365242004, "p95": 0.2775976359844208 }, { "label": "completed_group_2", "mean": 0.16849923133850098, "median": 0.16119162738323212, "p05": 0.1375115066766739, "p95": 0.2266949564218521 }, { "label": "completed_group_3", "mean": 0.18937936425209045, "median": 0.18853023648262024, "p05": 0.1678447425365448, "p95": 0.21501585841178894 }, { "label": "current_group_partial", "mean": 0.26560431718826294, "median": 0.28881219029426575, "p05": 0.117766834795475, "p95": 0.34617751836776733 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.6027262210845947, "group": 4, "latest_source_mass": 0.22770702838897705, "layer": 15, "max_source_mass": 0.22770702838897705, "mixer_index": 29, "normalized_entropy": 0.9958297916945706, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.20452985167503357, "median": 0.20441509783267975, "p05": 0.17931760847568512, "p95": 0.2289508432149887 }, { "label": "completed_group_1", "mean": 0.18702909350395203, "median": 0.186691552400589, "p05": 0.1675943285226822, "p95": 0.20997506380081177 }, { "label": "completed_group_2", "mean": 0.17977532744407654, "median": 0.177423357963562, "p05": 0.1569271981716156, "p95": 0.20937731862068176 }, { "label": "completed_group_3", "mean": 0.20095868408679962, "median": 0.2005838304758072, "p05": 0.1845097541809082, "p95": 0.21870870888233185 }, { "label": "current_group_partial", "mean": 0.22770702838897705, "median": 0.22757767140865326, "p05": 0.18680104613304138, "p95": 0.2673209607601166 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.577937126159668, "group": 4, "latest_source_mass": 0.2687118649482727, "layer": 16, "max_source_mass": 0.2687118649482727, "mixer_index": 30, "normalized_entropy": 0.9804274610215993, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1962222456932068, "median": 0.19565686583518982, "p05": 0.16995516419410706, "p95": 0.22565382719039917 }, { "label": "completed_group_1", "mean": 0.1818992793560028, "median": 0.16701740026474, "p05": 0.138178750872612, "p95": 0.2653764486312866 }, { "label": "completed_group_2", "mean": 0.17167484760284424, "median": 0.16360890865325928, "p05": 0.14424335956573486, "p95": 0.22631293535232544 }, { "label": "completed_group_3", "mean": 0.18149182200431824, "median": 0.18099229037761688, "p05": 0.16264864802360535, "p95": 0.20300358533859253 }, { "label": "current_group_partial", "mean": 0.2687118649482727, "median": 0.29608893394470215, "p05": 0.1254507452249527, "p95": 0.339657723903656 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.5977532863616943, "group": 4, "latest_source_mass": 0.23852142691612244, "layer": 16, "max_source_mass": 0.23852142691612244, "mixer_index": 31, "normalized_entropy": 0.9927399336239481, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.21533392369747162, "median": 0.21377520263195038, "p05": 0.1841924786567688, "p95": 0.24801591038703918 }, { "label": "completed_group_1", "mean": 0.18402470648288727, "median": 0.18508511781692505, "p05": 0.15855011343955994, "p95": 0.21014195680618286 }, { "label": "completed_group_2", "mean": 0.17019468545913696, "median": 0.16691477596759796, "p05": 0.1446647047996521, "p95": 0.20511509478092194 }, { "label": "completed_group_3", "mean": 0.1919252574443817, "median": 0.191789448261261, "p05": 0.17424410581588745, "p95": 0.20975348353385925 }, { "label": "current_group_partial", "mean": 0.23852142691612244, "median": 0.24018850922584534, "p05": 0.18989041447639465, "p95": 0.27930569648742676 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.578121542930603, "group": 5, "latest_source_mass": 0.26854145526885986, "layer": 17, "max_source_mass": 0.26854145526885986, "mixer_index": 32, "normalized_entropy": 0.9805420456038999, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.20422279834747314, "median": 0.2033933699131012, "p05": 0.17994040250778198, "p95": 0.23156529664993286 }, { "label": "completed_group_1", "mean": 0.1835813671350479, "median": 0.17072421312332153, "p05": 0.14088554680347443, "p95": 0.26194027066230774 }, { "label": "completed_group_2", "mean": 0.16642534732818604, "median": 0.1602073311805725, "p05": 0.13873112201690674, "p95": 0.22039896249771118 }, { "label": "completed_group_3", "mean": 0.17722904682159424, "median": 0.17722415924072266, "p05": 0.1577582061290741, "p95": 0.19813896715641022 }, { "label": "completed_group_4", "mean": 0.26854145526885986, "median": 0.2872956693172455, "p05": 0.12619024515151978, "p95": 0.3464823067188263 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.782798409461975, "group": 5, "latest_source_mass": 0.19473549723625183, "layer": 17, "max_source_mass": 0.19473549723625183, "mixer_index": 33, "normalized_entropy": 0.99499873731939, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.15961387753486633, "median": 0.15747272968292236, "p05": 0.12699076533317566, "p95": 0.19599634408950806 }, { "label": "completed_group_1", "mean": 0.16475997865200043, "median": 0.16349202394485474, "p05": 0.1474122852087021, "p95": 0.1848660409450531 }, { "label": "completed_group_2", "mean": 0.14714640378952026, "median": 0.14494061470031738, "p05": 0.12754470109939575, "p95": 0.1751968413591385 }, { "label": "completed_group_3", "mean": 0.15768736600875854, "median": 0.15663693845272064, "p05": 0.13646987080574036, "p95": 0.1821092814207077 }, { "label": "completed_group_4", "mean": 0.1760568618774414, "median": 0.17550913989543915, "p05": 0.15776243805885315, "p95": 0.19570523500442505 }, { "label": "current_group_partial", "mean": 0.19473549723625183, "median": 0.1956680715084076, "p05": 0.14822278916835785, "p95": 0.22800904512405396 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.7707297801971436, "group": 5, "latest_source_mass": 0.20674791932106018, "layer": 18, "max_source_mass": 0.20674791932106018, "mixer_index": 34, "normalized_entropy": 0.9882631070787802, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.16306114196777344, "median": 0.16283683478832245, "p05": 0.13633866608142853, "p95": 0.19143974781036377 }, { "label": "completed_group_1", "mean": 0.1578725278377533, "median": 0.14938931167125702, "p05": 0.13056670129299164, "p95": 0.21463912725448608 }, { "label": "completed_group_2", "mean": 0.139655202627182, "median": 0.1338472068309784, "p05": 0.11858384311199188, "p95": 0.18817788362503052 }, { "label": "completed_group_3", "mean": 0.14299750328063965, "median": 0.14077699184417725, "p05": 0.12273441255092621, "p95": 0.17249858379364014 }, { "label": "completed_group_4", "mean": 0.18966571986675262, "median": 0.19467708468437195, "p05": 0.13032010197639465, "p95": 0.23041488230228424 }, { "label": "current_group_partial", "mean": 0.20674791932106018, "median": 0.21543893218040466, "p05": 0.11787950992584229, "p95": 0.24730797111988068 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.781312108039856, "group": 5, "latest_source_mass": 0.20407646894454956, "layer": 18, "max_source_mass": 0.20407646894454956, "mixer_index": 35, "normalized_entropy": 0.9941692167014471, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.15919369459152222, "median": 0.15759795904159546, "p05": 0.13659372925758362, "p95": 0.19095730781555176 }, { "label": "completed_group_1", "mean": 0.1684478521347046, "median": 0.16807237267494202, "p05": 0.15075388550758362, "p95": 0.1875023990869522 }, { "label": "completed_group_2", "mean": 0.14742717146873474, "median": 0.14584535360336304, "p05": 0.12703484296798706, "p95": 0.1739335060119629 }, { "label": "completed_group_3", "mean": 0.1539241522550583, "median": 0.1541770100593567, "p05": 0.13218382000923157, "p95": 0.1755600869655609 }, { "label": "completed_group_4", "mean": 0.1669306606054306, "median": 0.16647151112556458, "p05": 0.15041440725326538, "p95": 0.18558931350708008 }, { "label": "current_group_partial", "mean": 0.20407646894454956, "median": 0.20521500706672668, "p05": 0.1562502235174179, "p95": 0.24481025338172913 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.7736209630966187, "group": 5, "latest_source_mass": 0.2125461846590042, "layer": 19, "max_source_mass": 0.2125461846590042, "mixer_index": 36, "normalized_entropy": 0.9898767069782805, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.15695160627365112, "median": 0.15683278441429138, "p05": 0.12300770729780197, "p95": 0.196682408452034 }, { "label": "completed_group_1", "mean": 0.16567477583885193, "median": 0.16040010750293732, "p05": 0.14332951605319977, "p95": 0.20587366819381714 }, { "label": "completed_group_2", "mean": 0.1478763222694397, "median": 0.14480885863304138, "p05": 0.12841497361660004, "p95": 0.18284933269023895 }, { "label": "completed_group_3", "mean": 0.14320014417171478, "median": 0.14095112681388855, "p05": 0.12106460332870483, "p95": 0.17675133049488068 }, { "label": "completed_group_4", "mean": 0.17375096678733826, "median": 0.17515000700950623, "p05": 0.1435358226299286, "p95": 0.19906878471374512 }, { "label": "current_group_partial", "mean": 0.2125461846590042, "median": 0.2186175137758255, "p05": 0.12126382440328598, "p95": 0.2633163630962372 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.781569480895996, "group": 5, "latest_source_mass": 0.20039017498493195, "layer": 19, "max_source_mass": 0.20039017498493195, "mixer_index": 37, "normalized_entropy": 0.9943128592274447, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.15368077158927917, "median": 0.1525704562664032, "p05": 0.12359238415956497, "p95": 0.20425409078598022 }, { "label": "completed_group_1", "mean": 0.1725173443555832, "median": 0.17324355244636536, "p05": 0.1533699631690979, "p95": 0.1897408664226532 }, { "label": "completed_group_2", "mean": 0.14978237450122833, "median": 0.14767895638942719, "p05": 0.1299348920583725, "p95": 0.17630663514137268 }, { "label": "completed_group_3", "mean": 0.15674246847629547, "median": 0.15763753652572632, "p05": 0.13473035395145416, "p95": 0.17680563032627106 }, { "label": "completed_group_4", "mean": 0.16688686609268188, "median": 0.16692788898944855, "p05": 0.14797358214855194, "p95": 0.18507987260818481 }, { "label": "current_group_partial", "mean": 0.20039017498493195, "median": 0.20044702291488647, "p05": 0.1619955599308014, "p95": 0.23957164585590363 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.7793617248535156, "group": 5, "latest_source_mass": 0.20417512953281403, "layer": 20, "max_source_mass": 0.20417512953281403, "mixer_index": 38, "normalized_entropy": 0.9930806871193036, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1678246557712555, "median": 0.17157375812530518, "p05": 0.13025718927383423, "p95": 0.19556939601898193 }, { "label": "completed_group_1", "mean": 0.1683025360107422, "median": 0.16609728336334229, "p05": 0.15072931349277496, "p95": 0.19358061254024506 }, { "label": "completed_group_2", "mean": 0.14878366887569427, "median": 0.1468905806541443, "p05": 0.13322587311267853, "p95": 0.1712985783815384 }, { "label": "completed_group_3", "mean": 0.14351718127727509, "median": 0.1410684883594513, "p05": 0.12425544857978821, "p95": 0.1736954152584076 }, { "label": "completed_group_4", "mean": 0.16739682853221893, "median": 0.16813713312149048, "p05": 0.14868372678756714, "p95": 0.1844603419303894 }, { "label": "current_group_partial", "mean": 0.20417512953281403, "median": 0.20674002170562744, "p05": 0.14016404747962952, "p95": 0.2501453757286072 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.7772380113601685, "group": 5, "latest_source_mass": 0.19795814156532288, "layer": 20, "max_source_mass": 0.19795814156532288, "mixer_index": 39, "normalized_entropy": 0.9918954200509164, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.15367703139781952, "median": 0.15482746064662933, "p05": 0.11167150735855103, "p95": 0.2207922786474228 }, { "label": "completed_group_1", "mean": 0.18459689617156982, "median": 0.184708833694458, "p05": 0.160706028342247, "p95": 0.20910656452178955 }, { "label": "completed_group_2", "mean": 0.14832255244255066, "median": 0.1467352658510208, "p05": 0.12785059213638306, "p95": 0.17438356578350067 }, { "label": "completed_group_3", "mean": 0.15016385912895203, "median": 0.14991271495819092, "p05": 0.12576226890087128, "p95": 0.17565840482711792 }, { "label": "completed_group_4", "mean": 0.1652815043926239, "median": 0.16595271229743958, "p05": 0.14006288349628448, "p95": 0.18747010827064514 }, { "label": "current_group_partial", "mean": 0.19795814156532288, "median": 0.19790686666965485, "p05": 0.1500549614429474, "p95": 0.24641191959381104 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.780207633972168, "group": 6, "latest_source_mass": 0.20078188180923462, "layer": 21, "max_source_mass": 0.20078188180923462, "mixer_index": 40, "normalized_entropy": 0.9935527979875202, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.16951784491539001, "median": 0.1744978278875351, "p05": 0.12857238948345184, "p95": 0.2016112506389618 }, { "label": "completed_group_1", "mean": 0.17200809717178345, "median": 0.17267316579818726, "p05": 0.15072743594646454, "p95": 0.19161207973957062 }, { "label": "completed_group_2", "mean": 0.15038041770458221, "median": 0.14867597818374634, "p05": 0.13418853282928467, "p95": 0.17207789421081543 }, { "label": "completed_group_3", "mean": 0.14669056236743927, "median": 0.14433687925338745, "p05": 0.12725189328193665, "p95": 0.1738971769809723 }, { "label": "completed_group_4", "mean": 0.16062124073505402, "median": 0.16014984250068665, "p05": 0.14366847276687622, "p95": 0.17978575825691223 }, { "label": "completed_group_5", "mean": 0.20078188180923462, "median": 0.20047783851623535, "p05": 0.14466112852096558, "p95": 0.25333014130592346 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.937079668045044, "group": 6, "latest_source_mass": 0.16695083677768707, "layer": 21, "max_source_mass": 0.16695083677768707, "mixer_index": 41, "normalized_entropy": 0.9954620304464951, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.13008031249046326, "median": 0.12692952156066895, "p05": 0.10487981885671616, "p95": 0.16793549060821533 }, { "label": "completed_group_1", "mean": 0.15163564682006836, "median": 0.15097364783287048, "p05": 0.13223227858543396, "p95": 0.17222966253757477 }, { "label": "completed_group_2", "mean": 0.1323973685503006, "median": 0.131024569272995, "p05": 0.11479418724775314, "p95": 0.15471340715885162 }, { "label": "completed_group_3", "mean": 0.13641127943992615, "median": 0.13619393110275269, "p05": 0.11877292394638062, "p95": 0.15510697662830353 }, { "label": "completed_group_4", "mean": 0.13806137442588806, "median": 0.1375710368156433, "p05": 0.11978909373283386, "p95": 0.15855638682842255 }, { "label": "completed_group_5", "mean": 0.1444631814956665, "median": 0.1438722014427185, "p05": 0.12040875852108002, "p95": 0.17076769471168518 }, { "label": "current_group_partial", "mean": 0.16695083677768707, "median": 0.17087092995643616, "p05": 0.13015522062778473, "p95": 0.195466548204422 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.939245343208313, "group": 6, "latest_source_mass": 0.16357660293579102, "layer": 22, "max_source_mass": 0.16357660293579102, "mixer_index": 42, "normalized_entropy": 0.9965749673230103, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.14692580699920654, "median": 0.1518530696630478, "p05": 0.11718790233135223, "p95": 0.16568630933761597 }, { "label": "completed_group_1", "mean": 0.14671877026557922, "median": 0.14763988554477692, "p05": 0.12195808440446854, "p95": 0.1661134660243988 }, { "label": "completed_group_2", "mean": 0.13336658477783203, "median": 0.13267168402671814, "p05": 0.12229838222265244, "p95": 0.14625124633312225 }, { "label": "completed_group_3", "mean": 0.1307726800441742, "median": 0.12948819994926453, "p05": 0.11789381504058838, "p95": 0.1479169726371765 }, { "label": "completed_group_4", "mean": 0.131200909614563, "median": 0.1309259533882141, "p05": 0.11869271099567413, "p95": 0.14432916045188904 }, { "label": "completed_group_5", "mean": 0.147438645362854, "median": 0.14618173241615295, "p05": 0.13021595776081085, "p95": 0.16755904257297516 }, { "label": "current_group_partial", "mean": 0.16357660293579102, "median": 0.1623307466506958, "p05": 0.13464656472206116, "p95": 0.1969548761844635 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.9312366247177124, "group": 6, "latest_source_mass": 0.1666574329137802, "layer": 22, "max_source_mass": 0.16950437426567078, "mixer_index": 43, "normalized_entropy": 0.9924593001661848, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.12334204465150833, "median": 0.12106236815452576, "p05": 0.08947738260030746, "p95": 0.17906470596790314 }, { "label": "completed_group_1", "mean": 0.16950437426567078, "median": 0.17181579768657684, "p05": 0.1329171359539032, "p95": 0.19668330252170563 }, { "label": "completed_group_2", "mean": 0.13870283961296082, "median": 0.13818645477294922, "p05": 0.12228117883205414, "p95": 0.1556926816701889 }, { "label": "completed_group_3", "mean": 0.1300538182258606, "median": 0.13022829592227936, "p05": 0.1119839996099472, "p95": 0.14633683860301971 }, { "label": "completed_group_4", "mean": 0.12885478138923645, "median": 0.1284654438495636, "p05": 0.11224459111690521, "p95": 0.14672259986400604 }, { "label": "completed_group_5", "mean": 0.14288470149040222, "median": 0.14101536571979523, "p05": 0.11797811090946198, "p95": 0.17301785945892334 }, { "label": "current_group_partial", "mean": 0.1666574329137802, "median": 0.1637348234653473, "p05": 0.12865713238716125, "p95": 0.20903055369853973 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9345850944519043, "group": 6, "latest_source_mass": 0.16773909330368042, "layer": 23, "max_source_mass": 0.16773909330368042, "mixer_index": 44, "normalized_entropy": 0.9941800732120613, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.1465124636888504, "median": 0.14716467261314392, "p05": 0.12261509895324707, "p95": 0.17156414687633514 }, { "label": "completed_group_1", "mean": 0.153631791472435, "median": 0.159563809633255, "p05": 0.10715117305517197, "p95": 0.18623968958854675 }, { "label": "completed_group_2", "mean": 0.13159149885177612, "median": 0.13108083605766296, "p05": 0.11123912036418915, "p95": 0.1546429693698883 }, { "label": "completed_group_3", "mean": 0.13120220601558685, "median": 0.13034585118293762, "p05": 0.11503913998603821, "p95": 0.1493607610464096 }, { "label": "completed_group_4", "mean": 0.12539167702198029, "median": 0.12449093163013458, "p05": 0.10983049124479294, "p95": 0.14342987537384033 }, { "label": "completed_group_5", "mean": 0.14393125474452972, "median": 0.1405339539051056, "p05": 0.1222267672419548, "p95": 0.17515066266059875 }, { "label": "current_group_partial", "mean": 0.16773909330368042, "median": 0.16424956917762756, "p05": 0.13410812616348267, "p95": 0.21529987454414368 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.9284589290618896, "group": 6, "latest_source_mass": 0.16532829403877258, "layer": 23, "max_source_mass": 0.169878289103508, "mixer_index": 45, "normalized_entropy": 0.9910318469730498, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.12412484735250473, "median": 0.12159939110279083, "p05": 0.08492134511470795, "p95": 0.18361014127731323 }, { "label": "completed_group_1", "mean": 0.169878289103508, "median": 0.17313054203987122, "p05": 0.1298009753227234, "p95": 0.1993248164653778 }, { "label": "completed_group_2", "mean": 0.13807371258735657, "median": 0.13714420795440674, "p05": 0.12058109790086746, "p95": 0.15811002254486084 }, { "label": "completed_group_3", "mean": 0.13034150004386902, "median": 0.13021467626094818, "p05": 0.11215148121118546, "p95": 0.14708518981933594 }, { "label": "completed_group_4", "mean": 0.13028821349143982, "median": 0.13050158321857452, "p05": 0.1127624660730362, "p95": 0.14778561890125275 }, { "label": "completed_group_5", "mean": 0.1419651210308075, "median": 0.1401965618133545, "p05": 0.11490530520677567, "p95": 0.17368361353874207 }, { "label": "current_group_partial", "mean": 0.16532829403877258, "median": 0.16024862229824066, "p05": 0.1169055625796318, "p95": 0.22413846850395203 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9329564571380615, "group": 6, "latest_source_mass": 0.17254900932312012, "layer": 24, "max_source_mass": 0.17254900932312012, "mixer_index": 46, "normalized_entropy": 0.9933431191961559, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1454196274280548, "median": 0.145169198513031, "p05": 0.12175656855106354, "p95": 0.17354263365268707 }, { "label": "completed_group_1", "mean": 0.1566689908504486, "median": 0.16300347447395325, "p05": 0.11168468743562698, "p95": 0.18563294410705566 }, { "label": "completed_group_2", "mean": 0.1286500096321106, "median": 0.12701794505119324, "p05": 0.11111085116863251, "p95": 0.15331649780273438 }, { "label": "completed_group_3", "mean": 0.12666445970535278, "median": 0.12615132331848145, "p05": 0.11176232993602753, "p95": 0.14301280677318573 }, { "label": "completed_group_4", "mean": 0.12782414257526398, "median": 0.1281316727399826, "p05": 0.11212349683046341, "p95": 0.14346082508563995 }, { "label": "completed_group_5", "mean": 0.14222374558448792, "median": 0.14010348916053772, "p05": 0.12181462347507477, "p95": 0.16952435672283173 }, { "label": "current_group_partial", "mean": 0.17254900932312012, "median": 0.1675749123096466, "p05": 0.1353597342967987, "p95": 0.2285366952419281 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.9259672164916992, "group": 6, "latest_source_mass": 0.16209754347801208, "layer": 24, "max_source_mass": 0.17791953682899475, "mixer_index": 47, "normalized_entropy": 0.989751360013567, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.12082186341285706, "median": 0.11626945436000824, "p05": 0.08509357273578644, "p95": 0.18624860048294067 }, { "label": "completed_group_1", "mean": 0.17791953682899475, "median": 0.1807878017425537, "p05": 0.1341632455587387, "p95": 0.2101013958454132 }, { "label": "completed_group_2", "mean": 0.1424642950296402, "median": 0.14065685868263245, "p05": 0.12273216992616653, "p95": 0.16812381148338318 }, { "label": "completed_group_3", "mean": 0.13298648595809937, "median": 0.13262662291526794, "p05": 0.11336477845907211, "p95": 0.15321806073188782 }, { "label": "completed_group_4", "mean": 0.12929362058639526, "median": 0.12950879335403442, "p05": 0.1104152500629425, "p95": 0.1479472517967224 }, { "label": "completed_group_5", "mean": 0.1344166398048401, "median": 0.13280367851257324, "p05": 0.1057823970913887, "p95": 0.16791269183158875 }, { "label": "current_group_partial", "mean": 0.16209754347801208, "median": 0.15705879032611847, "p05": 0.1103038564324379, "p95": 0.22623871266841888 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.9201786518096924, "group": 7, "latest_source_mass": 0.15931037068367004, "layer": 25, "max_source_mass": 0.17355048656463623, "mixer_index": 48, "normalized_entropy": 0.9867766262187836, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.13756656646728516, "median": 0.14409756660461426, "p05": 0.07605770230293274, "p95": 0.1807018518447876 }, { "label": "completed_group_1", "mean": 0.17355048656463623, "median": 0.1850908100605011, "p05": 0.09468921273946762, "p95": 0.2254364788532257 }, { "label": "completed_group_2", "mean": 0.14306488633155823, "median": 0.1434939205646515, "p05": 0.12189669907093048, "p95": 0.16250990331172943 }, { "label": "completed_group_3", "mean": 0.13006147742271423, "median": 0.12742848694324493, "p05": 0.10484378784894943, "p95": 0.15863747894763947 }, { "label": "completed_group_4", "mean": 0.11658850312232971, "median": 0.11437725275754929, "p05": 0.09685036540031433, "p95": 0.14346592128276825 }, { "label": "completed_group_5", "mean": 0.139857679605484, "median": 0.13671082258224487, "p05": 0.10990621894598007, "p95": 0.17913517355918884 }, { "label": "completed_group_6", "mean": 0.15931037068367004, "median": 0.15141257643699646, "p05": 0.10549820214509964, "p95": 0.2391991913318634 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 2.043663501739502, "group": 7, "latest_source_mass": 0.08876943588256836, "layer": 25, "max_source_mass": 0.17510183155536652, "mixer_index": 49, "normalized_entropy": 0.9827943997351177, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.10298450291156769, "median": 0.09445159137248993, "p05": 0.06719416379928589, "p95": 0.17596006393432617 }, { "label": "completed_group_1", "mean": 0.17510183155536652, "median": 0.1808546781539917, "p05": 0.12213055044412613, "p95": 0.21415141224861145 }, { "label": "completed_group_2", "mean": 0.1414714753627777, "median": 0.14097586274147034, "p05": 0.11696422845125198, "p95": 0.16771599650382996 }, { "label": "completed_group_3", "mean": 0.12628035247325897, "median": 0.12579765915870667, "p05": 0.10614679008722305, "p95": 0.14692939817905426 }, { "label": "completed_group_4", "mean": 0.11767096817493439, "median": 0.11719106882810593, "p05": 0.10052644461393356, "p95": 0.13653282821178436 }, { "label": "completed_group_5", "mean": 0.12069195508956909, "median": 0.11935321986675262, "p05": 0.09329675883054733, "p95": 0.15252181887626648 }, { "label": "completed_group_6", "mean": 0.12702946364879608, "median": 0.12134957313537598, "p05": 0.08377101272344589, "p95": 0.18232685327529907 }, { "label": "current_group_partial", "mean": 0.08876943588256836, "median": 0.07879889756441116, "p05": 0.06865477561950684, "p95": 0.15027400851249695 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.0607244968414307, "group": 7, "latest_source_mass": 0.11456632614135742, "layer": 26, "max_source_mass": 0.1498526930809021, "mixer_index": 50, "normalized_entropy": 0.9909990040771789, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1220984011888504, "median": 0.12646546959877014, "p05": 0.07202506065368652, "p95": 0.15947024524211884 }, { "label": "completed_group_1", "mean": 0.1498526930809021, "median": 0.15748712420463562, "p05": 0.08880092203617096, "p95": 0.19019576907157898 }, { "label": "completed_group_2", "mean": 0.12913095951080322, "median": 0.12894359230995178, "p05": 0.1061355471611023, "p95": 0.15167440474033356 }, { "label": "completed_group_3", "mean": 0.1179434061050415, "median": 0.118401437997818, "p05": 0.09755907952785492, "p95": 0.13774089515209198 }, { "label": "completed_group_4", "mean": 0.10775718837976456, "median": 0.10657531023025513, "p05": 0.09242597222328186, "p95": 0.12714339792728424 }, { "label": "completed_group_5", "mean": 0.12020602822303772, "median": 0.11893786489963531, "p05": 0.10092492401599884, "p95": 0.14495709538459778 }, { "label": "completed_group_6", "mean": 0.13844498991966248, "median": 0.13359181582927704, "p05": 0.10005055367946625, "p95": 0.1863655149936676 }, { "label": "current_group_partial", "mean": 0.11456632614135742, "median": 0.10762587189674377, "p05": 0.08914398401975632, "p95": 0.16366823017597198 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.0388383865356445, "group": 7, "latest_source_mass": 0.08920245617628098, "layer": 26, "max_source_mass": 0.18260905146598816, "mixer_index": 51, "normalized_entropy": 0.9804740098096767, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.10253133624792099, "median": 0.09456855058670044, "p05": 0.07405152916908264, "p95": 0.16842687129974365 }, { "label": "completed_group_1", "mean": 0.18260905146598816, "median": 0.19173294305801392, "p05": 0.11063728481531143, "p95": 0.22958147525787354 }, { "label": "completed_group_2", "mean": 0.14208021759986877, "median": 0.14284399151802063, "p05": 0.10792264342308044, "p95": 0.17174816131591797 }, { "label": "completed_group_3", "mean": 0.12606358528137207, "median": 0.12572120130062103, "p05": 0.10405182838439941, "p95": 0.14868314564228058 }, { "label": "completed_group_4", "mean": 0.11357533931732178, "median": 0.11289471387863159, "p05": 0.09577979147434235, "p95": 0.13378244638442993 }, { "label": "completed_group_5", "mean": 0.11596501618623734, "median": 0.1148330420255661, "p05": 0.09200432896614075, "p95": 0.1449994146823883 }, { "label": "completed_group_6", "mean": 0.12797299027442932, "median": 0.12260732054710388, "p05": 0.08455555140972137, "p95": 0.1881599724292755 }, { "label": "current_group_partial", "mean": 0.08920245617628098, "median": 0.07755134999752045, "p05": 0.066501684486866, "p95": 0.1560288667678833 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.0634398460388184, "group": 7, "latest_source_mass": 0.12262405455112457, "layer": 27, "max_source_mass": 0.15090109407901764, "mixer_index": 52, "normalized_entropy": 0.9923048110176299, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.12216806411743164, "median": 0.12567293643951416, "p05": 0.08283672481775284, "p95": 0.1497604101896286 }, { "label": "completed_group_1", "mean": 0.15090109407901764, "median": 0.15809711813926697, "p05": 0.09385120868682861, "p95": 0.1825941503047943 }, { "label": "completed_group_2", "mean": 0.12766355276107788, "median": 0.12779396772384644, "p05": 0.10599278658628464, "p95": 0.14822138845920563 }, { "label": "completed_group_3", "mean": 0.1134132444858551, "median": 0.11286742240190506, "p05": 0.09987462311983109, "p95": 0.12846460938453674 }, { "label": "completed_group_4", "mean": 0.10988184064626694, "median": 0.10929364711046219, "p05": 0.09590980410575867, "p95": 0.12525902688503265 }, { "label": "completed_group_5", "mean": 0.11335283517837524, "median": 0.11079756915569305, "p05": 0.09715084731578827, "p95": 0.14056096971035004 }, { "label": "completed_group_6", "mean": 0.13999530673027039, "median": 0.1355634331703186, "p05": 0.10303047299385071, "p95": 0.18968813121318817 }, { "label": "current_group_partial", "mean": 0.12262405455112457, "median": 0.11672497540712357, "p05": 0.09879282861948013, "p95": 0.16369464993476868 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.0386834144592285, "group": 7, "latest_source_mass": 0.08390261977910995, "layer": 27, "max_source_mass": 0.17931805551052094, "mixer_index": 53, "normalized_entropy": 0.9803994839943029, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.10029368102550507, "median": 0.09128708392381668, "p05": 0.07227826118469238, "p95": 0.17348244786262512 }, { "label": "completed_group_1", "mean": 0.17931805551052094, "median": 0.1862885057926178, "p05": 0.11921355128288269, "p95": 0.22246700525283813 }, { "label": "completed_group_2", "mean": 0.14185777306556702, "median": 0.14203041791915894, "p05": 0.10853314399719238, "p95": 0.1728457659482956 }, { "label": "completed_group_3", "mean": 0.13114282488822937, "median": 0.1317460834980011, "p05": 0.10662781447172165, "p95": 0.15405671298503876 }, { "label": "completed_group_4", "mean": 0.11872631311416626, "median": 0.11834143102169037, "p05": 0.09892343729734421, "p95": 0.1395292729139328 }, { "label": "completed_group_5", "mean": 0.11815309524536133, "median": 0.11647512018680573, "p05": 0.09220993518829346, "p95": 0.14916184544563293 }, { "label": "completed_group_6", "mean": 0.12660562992095947, "median": 0.12275825440883636, "p05": 0.08448901027441025, "p95": 0.1805192083120346 }, { "label": "current_group_partial", "mean": 0.08390261977910995, "median": 0.07283803820610046, "p05": 0.06168072670698166, "p95": 0.14687296748161316 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.0626261234283447, "group": 7, "latest_source_mass": 0.1115579903125763, "layer": 28, "max_source_mass": 0.15398627519607544, "mixer_index": 54, "normalized_entropy": 0.9919134931593667, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.11644256114959717, "median": 0.12025526911020279, "p05": 0.07829730212688446, "p95": 0.1473471075296402 }, { "label": "completed_group_1", "mean": 0.15398627519607544, "median": 0.1615961492061615, "p05": 0.09488844126462936, "p95": 0.18759915232658386 }, { "label": "completed_group_2", "mean": 0.13596117496490479, "median": 0.13718228042125702, "p05": 0.11204125732183456, "p95": 0.15343619883060455 }, { "label": "completed_group_3", "mean": 0.12036904692649841, "median": 0.1201494038105011, "p05": 0.10638508945703506, "p95": 0.13539429008960724 }, { "label": "completed_group_4", "mean": 0.10964121669530869, "median": 0.10774455964565277, "p05": 0.09591315686702728, "p95": 0.1285344362258911 }, { "label": "completed_group_5", "mean": 0.11886264383792877, "median": 0.11775418370962143, "p05": 0.0997074693441391, "p95": 0.14473959803581238 }, { "label": "completed_group_6", "mean": 0.13317911326885223, "median": 0.12975312769412994, "p05": 0.09947147965431213, "p95": 0.1780128926038742 }, { "label": "current_group_partial", "mean": 0.1115579903125763, "median": 0.1049065813422203, "p05": 0.0882733166217804, "p95": 0.15690873563289642 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.020051956176758, "group": 7, "latest_source_mass": 0.07486395537853241, "layer": 28, "max_source_mass": 0.1913527548313141, "mixer_index": 55, "normalized_entropy": 0.9714396465047528, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.09590239822864532, "median": 0.08437805622816086, "p05": 0.062284864485263824, "p95": 0.19059056043624878 }, { "label": "completed_group_1", "mean": 0.1913527548313141, "median": 0.19941553473472595, "p05": 0.11809638142585754, "p95": 0.24551606178283691 }, { "label": "completed_group_2", "mean": 0.14906750619411469, "median": 0.14890097081661224, "p05": 0.1085248589515686, "p95": 0.18817238509655 }, { "label": "completed_group_3", "mean": 0.1341754049062729, "median": 0.13468942046165466, "p05": 0.10614670813083649, "p95": 0.16129878163337708 }, { "label": "completed_group_4", "mean": 0.11847006529569626, "median": 0.11653131991624832, "p05": 0.09301862120628357, "p95": 0.14944709837436676 }, { "label": "completed_group_5", "mean": 0.11603337526321411, "median": 0.11465917527675629, "p05": 0.08672616630792618, "p95": 0.1505887657403946 }, { "label": "completed_group_6", "mean": 0.12013453245162964, "median": 0.11430312693119049, "p05": 0.07354772090911865, "p95": 0.18300247192382812 }, { "label": "current_group_partial", "mean": 0.07486395537853241, "median": 0.06415845453739166, "p05": 0.052880410104990005, "p95": 0.13596120476722717 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 2.0452136993408203, "group": 8, "latest_source_mass": 0.09500804543495178, "layer": 29, "max_source_mass": 0.15673108398914337, "mixer_index": 56, "normalized_entropy": 0.9835398871990577, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.11420649290084839, "median": 0.11760550737380981, "p05": 0.0455118790268898, "p95": 0.1930246502161026 }, { "label": "completed_group_1", "mean": 0.15673108398914337, "median": 0.1630343794822693, "p05": 0.0941019207239151, "p95": 0.2044450342655182 }, { "label": "completed_group_2", "mean": 0.13583534955978394, "median": 0.1358843892812729, "p05": 0.10828851163387299, "p95": 0.16276802122592926 }, { "label": "completed_group_3", "mean": 0.12758757174015045, "median": 0.12776434421539307, "p05": 0.10106872022151947, "p95": 0.1536240577697754 }, { "label": "completed_group_4", "mean": 0.1112617775797844, "median": 0.10795636475086212, "p05": 0.0855436623096466, "p95": 0.1458887755870819 }, { "label": "completed_group_5", "mean": 0.13417255878448486, "median": 0.13457609713077545, "p05": 0.10089332610368729, "p95": 0.16806170344352722 }, { "label": "completed_group_6", "mean": 0.12519711256027222, "median": 0.11858057975769043, "p05": 0.08283591270446777, "p95": 0.18291912972927094 }, { "label": "completed_group_7", "mean": 0.09500804543495178, "median": 0.086635060608387, "p05": 0.07067855447530746, "p95": 0.14996609091758728 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 2.1403071880340576, "group": 8, "latest_source_mass": 0.10776971280574799, "layer": 29, "max_source_mass": 0.1673685610294342, "mixer_index": 57, "normalized_entropy": 0.9740957797899907, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.08488726615905762, "median": 0.07340645045042038, "p05": 0.057354122400283813, "p95": 0.1699385792016983 }, { "label": "completed_group_1", "mean": 0.1673685610294342, "median": 0.17044895887374878, "p05": 0.10701967775821686, "p95": 0.213656947016716 }, { "label": "completed_group_2", "mean": 0.13726133108139038, "median": 0.13688170909881592, "p05": 0.09961593151092529, "p95": 0.17454354465007782 }, { "label": "completed_group_3", "mean": 0.12220048904418945, "median": 0.12267022579908371, "p05": 0.09608018398284912, "p95": 0.14777429401874542 }, { "label": "completed_group_4", "mean": 0.10616929084062576, "median": 0.10591423511505127, "p05": 0.08670090138912201, "p95": 0.12526394426822662 }, { "label": "completed_group_5", "mean": 0.1037440150976181, "median": 0.10274936258792877, "p05": 0.08146549016237259, "p95": 0.13024258613586426 }, { "label": "completed_group_6", "mean": 0.10401669144630432, "median": 0.10373582690954208, "p05": 0.06632979214191437, "p95": 0.14398667216300964 }, { "label": "completed_group_7", "mean": 0.06658264249563217, "median": 0.05958016216754913, "p05": 0.04998533055186272, "p95": 0.10914435982704163 }, { "label": "current_group_partial", "mean": 0.10776971280574799, "median": 0.09594395756721497, "p05": 0.05985837057232857, "p95": 0.1909799873828888 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.18704891204834, "group": 8, "latest_source_mass": 0.1059001013636589, "layer": 30, "max_source_mass": 0.12684623897075653, "mixer_index": 58, "normalized_entropy": 0.9953688551489734, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.10885682702064514, "median": 0.11496149003505707, "p05": 0.06692696362733841, "p95": 0.13488291203975677 }, { "label": "completed_group_1", "mean": 0.12684623897075653, "median": 0.12977978587150574, "p05": 0.0880269706249237, "p95": 0.15352922677993774 }, { "label": "completed_group_2", "mean": 0.11236055195331573, "median": 0.1118544340133667, "p05": 0.0995602011680603, "p95": 0.12739220261573792 }, { "label": "completed_group_3", "mean": 0.10673344880342484, "median": 0.10557802021503448, "p05": 0.09419171512126923, "p95": 0.12314252555370331 }, { "label": "completed_group_4", "mean": 0.10181289911270142, "median": 0.10104846954345703, "p05": 0.0905531719326973, "p95": 0.1154235228896141 }, { "label": "completed_group_5", "mean": 0.11441421508789062, "median": 0.11405807733535767, "p05": 0.1015545204281807, "p95": 0.12890149652957916 }, { "label": "completed_group_6", "mean": 0.11840946972370148, "median": 0.11774120479822159, "p05": 0.09769279509782791, "p95": 0.1386050581932068 }, { "label": "completed_group_7", "mean": 0.10466624796390533, "median": 0.10123022645711899, "p05": 0.08714763075113297, "p95": 0.1335311383008957 }, { "label": "current_group_partial", "mean": 0.1059001013636589, "median": 0.10151755809783936, "p05": 0.08478552103042603, "p95": 0.14096680283546448 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.1283018589019775, "group": 8, "latest_source_mass": 0.07073106616735458, "layer": 30, "max_source_mass": 0.1812124252319336, "mixer_index": 59, "normalized_entropy": 0.9686319190376982, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.08667369186878204, "median": 0.07598486542701721, "p05": 0.0599454902112484, "p95": 0.17076340317726135 }, { "label": "completed_group_1", "mean": 0.1812124252319336, "median": 0.18718186020851135, "p05": 0.11695106327533722, "p95": 0.22937621176242828 }, { "label": "completed_group_2", "mean": 0.14178472757339478, "median": 0.1410643458366394, "p05": 0.10069074481725693, "p95": 0.18480442464351654 }, { "label": "completed_group_3", "mean": 0.12440058588981628, "median": 0.12452012300491333, "p05": 0.09520045667886734, "p95": 0.15427032113075256 }, { "label": "completed_group_4", "mean": 0.11160320043563843, "median": 0.11176156997680664, "p05": 0.08930732309818268, "p95": 0.1340266317129135 }, { "label": "completed_group_5", "mean": 0.10330478101968765, "median": 0.10138857364654541, "p05": 0.07821988314390182, "p95": 0.13451263308525085 }, { "label": "completed_group_6", "mean": 0.11197599023580551, "median": 0.11038671433925629, "p05": 0.06996595859527588, "p95": 0.1621071994304657 }, { "label": "completed_group_7", "mean": 0.06831350922584534, "median": 0.06078982725739479, "p05": 0.049278710037469864, "p95": 0.11596570909023285 }, { "label": "current_group_partial", "mean": 0.07073106616735458, "median": 0.06323730200529099, "p05": 0.051058635115623474, "p95": 0.11510380357503891 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.173035144805908, "group": 8, "latest_source_mass": 0.097323939204216, "layer": 31, "max_source_mass": 0.13639645278453827, "mixer_index": 60, "normalized_entropy": 0.9889909148205337, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.10752685368061066, "median": 0.11246659606695175, "p05": 0.04835075885057449, "p95": 0.15864257514476776 }, { "label": "completed_group_1", "mean": 0.13639645278453827, "median": 0.1424846053123474, "p05": 0.0804441049695015, "p95": 0.17550095915794373 }, { "label": "completed_group_2", "mean": 0.11839058995246887, "median": 0.11810846626758575, "p05": 0.0967480018734932, "p95": 0.13973747193813324 }, { "label": "completed_group_3", "mean": 0.11235916614532471, "median": 0.112240731716156, "p05": 0.09452164173126221, "p95": 0.12995943427085876 }, { "label": "completed_group_4", "mean": 0.10138258337974548, "median": 0.09962283074855804, "p05": 0.08511637151241302, "p95": 0.12233416736125946 }, { "label": "completed_group_5", "mean": 0.11953261494636536, "median": 0.12015771865844727, "p05": 0.09630562365055084, "p95": 0.14168336987495422 }, { "label": "completed_group_6", "mean": 0.11256720125675201, "median": 0.11007197946310043, "p05": 0.0816735252737999, "p95": 0.14915722608566284 }, { "label": "completed_group_7", "mean": 0.09452059864997864, "median": 0.0890384241938591, "p05": 0.07612448185682297, "p95": 0.13178984820842743 }, { "label": "current_group_partial", "mean": 0.097323939204216, "median": 0.0898042768239975, "p05": 0.07185541093349457, "p95": 0.14817608892917633 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.1028335094451904, "group": 8, "latest_source_mass": 0.06014348566532135, "layer": 31, "max_source_mass": 0.19083696603775024, "mixer_index": 61, "normalized_entropy": 0.9570407736811942, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.08200174570083618, "median": 0.06842087209224701, "p05": 0.05465565621852875, "p95": 0.1860819309949875 }, { "label": "completed_group_1", "mean": 0.19083696603775024, "median": 0.19397340714931488, "p05": 0.1252523958683014, "p95": 0.24377956986427307 }, { "label": "completed_group_2", "mean": 0.15512439608573914, "median": 0.15396007895469666, "p05": 0.10948964953422546, "p95": 0.2031564563512802 }, { "label": "completed_group_3", "mean": 0.1322997808456421, "median": 0.131678968667984, "p05": 0.1001092940568924, "p95": 0.16698354482650757 }, { "label": "completed_group_4", "mean": 0.11209174990653992, "median": 0.11124030500650406, "p05": 0.08621159940958023, "p95": 0.1389508843421936 }, { "label": "completed_group_5", "mean": 0.10435720533132553, "median": 0.10206517577171326, "p05": 0.07588012516498566, "p95": 0.13983221352100372 }, { "label": "completed_group_6", "mean": 0.1024213582277298, "median": 0.10071991384029388, "p05": 0.06045639514923096, "p95": 0.15181156992912292 }, { "label": "completed_group_7", "mean": 0.06072331964969635, "median": 0.05212578922510147, "p05": 0.04197835922241211, "p95": 0.10935051739215851 }, { "label": "current_group_partial", "mean": 0.06014348566532135, "median": 0.054720573127269745, "p05": 0.043509967625141144, "p95": 0.09508825838565826 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.147671937942505, "group": 8, "latest_source_mass": 0.07905417680740356, "layer": 32, "max_source_mass": 0.14984291791915894, "mixer_index": 62, "normalized_entropy": 0.9774476219204733, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.09926988929510117, "median": 0.09770013391971588, "p05": 0.04949922859668732, "p95": 0.1754719465970993 }, { "label": "completed_group_1", "mean": 0.14984291791915894, "median": 0.1559048891067505, "p05": 0.08648592233657837, "p95": 0.2032528817653656 }, { "label": "completed_group_2", "mean": 0.12505242228507996, "median": 0.12578052282333374, "p05": 0.09032565355300903, "p95": 0.15844053030014038 }, { "label": "completed_group_3", "mean": 0.13004425168037415, "median": 0.12868037819862366, "p05": 0.10237257927656174, "p95": 0.1618403047323227 }, { "label": "completed_group_4", "mean": 0.1122550517320633, "median": 0.10850167274475098, "p05": 0.08462762087583542, "p95": 0.15187090635299683 }, { "label": "completed_group_5", "mean": 0.12226949632167816, "median": 0.12273561954498291, "p05": 0.08938803523778915, "p95": 0.15480002760887146 }, { "label": "completed_group_6", "mean": 0.10689134150743484, "median": 0.10258689522743225, "p05": 0.07286856323480606, "p95": 0.15216295421123505 }, { "label": "completed_group_7", "mean": 0.07532045990228653, "median": 0.06964509189128876, "p05": 0.054946303367614746, "p95": 0.11525978893041611 }, { "label": "current_group_partial", "mean": 0.07905417680740356, "median": 0.072466179728508, "p05": 0.05414725840091705, "p95": 0.11476384103298187 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.109609603881836, "group": 8, "latest_source_mass": 0.06053228676319122, "layer": 32, "max_source_mass": 0.18956288695335388, "mixer_index": 63, "normalized_entropy": 0.9601247071609755, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.08386656641960144, "median": 0.07061353325843811, "p05": 0.055651482194662094, "p95": 0.18354055285453796 }, { "label": "completed_group_1", "mean": 0.18956288695335388, "median": 0.19494932889938354, "p05": 0.12378176301717758, "p95": 0.24272672832012177 }, { "label": "completed_group_2", "mean": 0.1487923562526703, "median": 0.14935529232025146, "p05": 0.10487234592437744, "p95": 0.19193747639656067 }, { "label": "completed_group_3", "mean": 0.12990039587020874, "median": 0.1300441324710846, "p05": 0.09967920184135437, "p95": 0.160600483417511 }, { "label": "completed_group_4", "mean": 0.11159682273864746, "median": 0.11003521829843521, "p05": 0.08588480204343796, "p95": 0.14072611927986145 }, { "label": "completed_group_5", "mean": 0.10738209635019302, "median": 0.10594583302736282, "p05": 0.0788765475153923, "p95": 0.14147546887397766 }, { "label": "completed_group_6", "mean": 0.10535432398319244, "median": 0.10358455777168274, "p05": 0.06328895688056946, "p95": 0.15448220074176788 }, { "label": "completed_group_7", "mean": 0.0630122721195221, "median": 0.05402897298336029, "p05": 0.044587839394807816, "p95": 0.11368448287248611 }, { "label": "current_group_partial", "mean": 0.06053228676319122, "median": 0.055282287299633026, "p05": 0.04416987672448158, "p95": 0.09472700953483582 } ], "sources": 9 }, { "branch": "output", "entropy_mean": 2.1622862815856934, "group": null, "latest_source_mass": 0.12579965591430664, "layer": null, "max_source_mass": 0.14956718683242798, "mixer_index": null, "normalized_entropy": 0.9840988963481907, "offset": null, "source_summaries": [ { "label": "embedding", "mean": 0.12356588989496231, "median": 0.12339343130588531, "p05": 0.0884157195687294, "p95": 0.16069066524505615 }, { "label": "completed_group_1", "mean": 0.09235234558582306, "median": 0.09365912526845932, "p05": 0.06739562749862671, "p95": 0.11328895390033722 }, { "label": "completed_group_2", "mean": 0.08736803382635117, "median": 0.08676370978355408, "p05": 0.07290066033601761, "p95": 0.10374386608600616 }, { "label": "completed_group_3", "mean": 0.08735492825508118, "median": 0.08669611811637878, "p05": 0.07205022871494293, "p95": 0.10462935268878937 }, { "label": "completed_group_4", "mean": 0.08889827132225037, "median": 0.08936247229576111, "p05": 0.07074585556983948, "p95": 0.10606974363327026 }, { "label": "completed_group_5", "mean": 0.10504580289125443, "median": 0.10402922332286835, "p05": 0.0790465772151947, "p95": 0.13349474966526031 }, { "label": "completed_group_6", "mean": 0.14004790782928467, "median": 0.13968253135681152, "p05": 0.10069862753152847, "p95": 0.1819821298122406 }, { "label": "completed_group_7", "mean": 0.14956718683242798, "median": 0.14930106699466705, "p05": 0.12316055595874786, "p95": 0.17985624074935913 }, { "label": "completed_group_8", "mean": 0.12579965591430664, "median": 0.12568724155426025, "p05": 0.07226289063692093, "p95": 0.17831499874591827 } ], "sources": 9 } ], "mode": "learned", "optimizer_state_after": "d138a51152c0ced0c4aa892596a1d6f5a083daa6c423038734049796569e3c34", "optimizer_state_before": "d138a51152c0ced0c4aa892596a1d6f5a083daa6c423038734049796569e3c34", "optimizer_state_unchanged": true, "positions": { "attention_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "981ba7bc004b8232ce5e00b2be2c80efd41f83b0edfc6e522ef32de2929ebcb1", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.010549778118729591, 0.013795096427202225, 0.012347182258963585, 0.012820021249353886, 0.01913420483469963, 0.018582463264465332, 0.013614931143820286, 0.013612493872642517, 0.019129201769828796, 0.02525762841105461, 0.027986859902739525, 0.014458909630775452, 0.018231205642223358, 0.016914349049329758, 0.022478405386209488, 0.03932488337159157, 0.03611915931105614, 0.0266753900796175, 0.018954744562506676, 0.014813725836575031, 0.013802599161863327, 0.016533223912119865, 0.016679473221302032, 0.01826235093176365, 0.015529165975749493, 0.011229476891458035, 0.008122270926833153, 0.008894859813153744, 0.009738311171531677, 0.007465044502168894, 0.008605006150901318, 0.009693782776594162 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 3.478636267573165e-05, "first_to_last_ratio": 5.2633491541899735, "imbalance_abs_log_ratio": 1.6607675454574897, "last_quartile_mean": 6.60916873584938e-06, "mean": 1.8399187894146962e-05, "normalized": [ 2.32572864627557, 2.093980749525082, 2.097253493167976, 1.9748047942119253, 1.9001199222002307, 1.7176970887283356, 1.5731226694511573, 1.442465715104865, 1.388173701809276, 1.2578149167122274, 1.163711179168628, 1.0739337627510765, 1.0625631931116428, 0.9721718406735991, 0.8978727300712377, 0.830314261296444, 0.8465768431906301, 0.7713093251343669, 0.7043450693146728, 0.6424476146199775, 0.6810764492568463, 0.62147201099682, 0.5661400138095133, 0.5212255519238714, 0.43050775556639687, 0.41135344603945795, 0.3923546966297886, 0.3784948438109706, 0.3251037643245192, 0.3141780758593552, 0.3117257919642976, 0.30996008329924213 ], "peak_layer": 1, "peak_normalized": 2.32572864627557, "population_cv": 0.5996620573317034, "rest_mean": 1.9884464530116895e-05, "spike_contrast": 0.5219498892811426, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.0378694059909322e-05, "top_five_layers": [ 1, 3, 2, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 4.279151835362427e-05, 3.852754525723867e-05, 3.858776108245365e-05, 3.633480446296744e-05, 3.496066346997395e-05, 3.160423148074187e-05, 2.8944179575773887e-05, 2.6540197723079473e-05, 2.5541268769302405e-05, 2.3142772988649085e-05, 2.141134064004291e-05, 1.9759509086725302e-05, 1.955029983946588e-05, 1.7887172361952253e-05, 1.65201290656114e-05, 1.527710810478311e-05, 1.5576326404698193e-05, 1.4191465197654907e-05, 1.295937727263663e-05, 1.1820514373539481e-05, 1.2531253560155164e-05, 1.1434580301283859e-05, 1.041651648847619e-05, 9.590126865077764e-06, 7.92099308455363e-06, 7.56856934458483e-06, 7.2190077844425105e-06, 6.963997748243855e-06, 5.981645244901301e-06, 5.7806214499578346e-06, 5.73550141780288e-06, 5.7030138123082e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.00013640912038681563, "first_to_last_ratio": 5.121983284590993, "imbalance_abs_log_ratio": 1.6335417243184733, "last_quartile_mean": 2.663209011188883e-05, "mean": 7.298096016938871e-05, "normalized": [ 2.2789977416441083, 2.0684974265127174, 2.076163899895869, 1.9536947342428674, 1.8791834481613054, 1.70558006750568, 1.5594206835801008, 1.4313066933158733, 1.382104599488904, 1.2645011290875958, 1.1686177713576213, 1.080869560986379, 1.070406696124593, 0.9820093608320563, 0.9088928309552108, 0.8405354206057669, 0.8564067293095593, 0.7807789902359659, 0.7130468773072498, 0.650906500520924, 0.6919540061414183, 0.6322816699001862, 0.5753535339386485, 0.5291429877926532, 0.43715672246183995, 0.41718055352967753, 0.3982548633280403, 0.38497320016438824, 0.33031538135673594, 0.31952371692646125, 0.31683848721825286, 0.31510371557135003 ], "peak_layer": 1, "peak_normalized": 2.2789977416441083, "population_cv": 0.5877415594416108, "rest_mean": 7.874945926973683e-05, "spike_contrast": 0.5311917747171685, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.183106502750888e-05, "top_five_layers": [ 1, 3, 2, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00016632344340905547, 0.00015096092829480767, 0.00015152043488342315, 0.00014258251758292317, 0.00013714461238123477, 0.00012447487097233534, 0.00011380801879568025, 0.00010445813677506521, 0.00010086732072522864, 9.228450653608888e-05, 8.528684702469036e-05, 7.888289837865159e-05, 7.811930845491588e-05, 7.166798604885116e-05, 6.633187149418518e-05, 6.134308205218986e-05, 6.25013854005374e-05, 5.698200038750656e-05, 5.2038845751667395e-05, 4.7503781388513744e-05, 5.04994677612558e-05, 4.614452336682007e-05, 4.198985334369354e-05, 3.861736331600696e-05, 3.190411734976806e-05, 3.044623736059293e-05, 2.9065022317809053e-05, 2.8095713787479326e-05, 2.4106733690132387e-05, 2.3319147658185102e-05, 2.3123177015804686e-05, 2.2996571715339087e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.12096897326409817, "first_to_last_ratio": 5.121983389958884, "imbalance_abs_log_ratio": 1.6335417448901706, "last_quartile_mean": 0.02361760358326137, "mean": 0.06472024595132098, "normalized": [ 2.2789977721075467, 2.0684973368733783, 2.0761638538672402, 1.953694825115509, 1.8791834311760194, 1.7055800372311585, 1.5594207388065806, 1.4313066964307424, 1.3821046167764135, 1.264501125605702, 1.1686178592805807, 1.080869532341778, 1.0704067558461303, 0.9820093835771068, 0.9088927809148353, 0.8405354027150204, 0.8564067368028002, 0.7807789687785609, 0.7130468636740094, 0.6509064717345558, 0.6919539840797562, 0.6322817027038486, 0.5753535315685364, 0.529143012125947, 0.4371566943170802, 0.41718053416692213, 0.39825484202638917, 0.3849732137717626, 0.3303153774147322, 0.31952373248220817, 0.3168384772300972, 0.31510370845705143 ], "peak_layer": 1, "peak_normalized": 2.2789977721075467, "population_cv": 0.5877415617202747, "rest_mean": 0.06983580868001338, "spike_contrast": 0.5311917756455903, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.03709620721638203, "top_five_layers": [ 1, 3, 2, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.147497296333313, 0.13387365639209747, 0.1343698352575302, 0.12644360959529877, 0.12162121385335922, 0.11038555949926376, 0.10092609375715256, 0.09263452142477036, 0.08945015072822571, 0.08183882385492325, 0.07563323527574539, 0.06995414197444916, 0.06927698850631714, 0.06355588883161545, 0.05882376432418823, 0.05439965799450874, 0.05542685464024544, 0.050532206892967224, 0.04614856839179993, 0.04212682694196701, 0.04478343203663826, 0.04092142730951309, 0.0372370220720768, 0.03424626588821411, 0.028292888775467873, 0.027000026777386665, 0.025775151327252388, 0.024915561079978943, 0.02137809246778488, 0.0206796545535326, 0.020505864173173904, 0.02039358951151371 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 7.711734099302703e-06, "first_to_last_ratio": 4.6783638296609995, "imbalance_abs_log_ratio": 1.54294843975527, "last_quartile_mean": 1.648382720986774e-06, "mean": 4.007230394620365e-06, "normalized": [ 3.358984576738256, 2.372641967350387, 2.13819011970407, 1.8393394452465484, 1.59553806661366, 1.4487556048903885, 1.2955565469325476, 1.3466327346665505, 1.2739273872501113, 1.0660089247937763, 1.0478478795998651, 1.0914936255170047, 1.0411021864111147, 0.9154927446436142, 0.8065881507886284, 0.7702055746502616, 0.8215637644294851, 0.7364762575936972, 0.6730206907422001, 0.6266437238096296, 0.6693956877918972, 0.6236755529871887, 0.5897829924931797, 0.560318828703654, 0.45883664783466677, 0.44720692888245495, 0.4388849180240322, 0.4318078866173612, 0.37877623882612776, 0.3751519735069292, 0.3795957753533195, 0.38055659660739233 ], "peak_layer": 1, "peak_normalized": 3.358984576738256, "population_cv": 0.6648021914060557, "rest_mean": 4.31860559678575e-06, "spike_contrast": 0.5385544594899636, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.3258043029272812e-06, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 1.346022509096656e-05, 9.50772300711833e-06, 8.568220437155105e-06, 7.3706569310161285e-06, 6.39368863630807e-06, 5.805497494293377e-06, 5.19159357281751e-06, 5.3962676247465424e-06, 5.1049205467279535e-06, 4.271743364370195e-06, 4.19896787207108e-06, 4.373866431706119e-06, 4.171936325292336e-06, 3.6685903523903107e-06, 3.232184553780826e-06, 3.0863911888445728e-06, 3.292195287940558e-06, 2.9512300443457207e-06, 2.6969489681505365e-06, 2.511105776648037e-06, 2.6824227461474948e-06, 2.4992116323119262e-06, 2.363396333748824e-06, 2.245326641059364e-06, 1.838664161368797e-06, 1.7920611981026013e-06, 1.7587129832463688e-06, 1.730353687889874e-06, 1.5178436569840414e-06, 1.5033203908387804e-06, 1.5211277286653058e-06, 1.5249779607984237e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 0.0001145691167039331, "first_to_last_ratio": 4.836936726468446, "imbalance_abs_log_ratio": 1.5762816126034256, "last_quartile_mean": 2.368629634474928e-05, "mean": 6.211562191538178e-05, "normalized": [ 2.348876988858354, 2.071301897195793, 2.0294561010843566, 1.9098237582754058, 1.8088773728255583, 1.6555764779765272, 1.5234729301569887, 1.4082090466002533, 1.3570526012658433, 1.2477509254620376, 1.1584754738367655, 1.0770687308168136, 1.0629673480990054, 0.9794260409949351, 0.9103157365029115, 0.8467425542641047, 0.8606040430432489, 0.7903153532861079, 0.7260518902950809, 0.6672536282299563, 0.7127906049054458, 0.6531196817841458, 0.5952239503818174, 0.548639546167493, 0.4562347554219186, 0.43716465149452993, 0.4212539905654889, 0.40928696694430294, 0.34099585105581787, 0.33116538282585856, 0.32832835587544684, 0.32617736350768683 ], "peak_layer": 1, "peak_normalized": 2.348876988858354, "population_cv": 0.5738572576504661, "rest_mean": 6.679497912248028e-05, "spike_contrast": 0.5516446517557004, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.6847092997049916e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00014590195496566594, 0.00012866020551882684, 0.00012606092786882073, 0.0001186298904940486, 0.00011235954298172146, 0.00010283716255798936, 9.463146852795035e-05, 8.747178071644157e-05, 8.429416629951447e-05, 7.750482473056763e-05, 7.195942453108728e-05, 6.690279406029731e-05, 6.602687790291384e-05, 6.0837657656520605e-05, 5.6544828112237155e-05, 5.259594036033377e-05, 5.3456955356523395e-05, 4.909092967864126e-05, 4.509916470851749e-05, 4.144687409279868e-05, 4.4275431719142944e-05, 4.0568935219198465e-05, 3.697270585689694e-05, 3.4079086617566645e-05, 2.8339305572444573e-05, 2.715475420700386e-05, 2.6166453608311713e-05, 2.542311449360568e-05, 2.1181169358897023e-05, 2.0570543711073697e-05, 2.0394320017658174e-05, 2.026070978899952e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 9.517959642835194e-05, "first_to_last_ratio": 4.053573868997912, "imbalance_abs_log_ratio": 1.39959892878961, "last_quartile_mean": 2.3480414938603644e-05, "mean": 5.345439518578132e-05, "normalized": [ 2.3487685774234603, 2.032002289203053, 1.963516724917696, 1.8356141378280442, 1.6872738480579543, 1.5647260343512426, 1.4530976571042717, 1.3596062267041897, 1.314646683688125, 1.2118086283799, 1.12751130576606, 1.057258217084054, 1.0421641943948246, 0.9643812108591268, 0.9028111172523068, 0.8481148430618181, 0.8638417335406758, 0.7968091313416481, 0.7398800228203574, 0.6885877184509301, 0.7528185246328142, 0.6965652282580993, 0.6386689529276455, 0.5954414082316423, 0.5183367890031948, 0.5027841940839927, 0.4926901944439643, 0.48501928420581064, 0.3867176094242744, 0.3783808574805794, 0.37618160958425234, 0.373975045493992 ], "peak_layer": 1, "peak_normalized": 2.3487685774234603, "population_cv": 0.531886342087635, "rest_mean": 5.701439672352367e-05, "spike_contrast": 0.6003814623868411, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.423038688197266e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00012555200373753905, 0.0001086194533854723, 0.00010495859896764159, 9.812164353206754e-05, 9.019220306072384e-05, 8.364148379769176e-05, 7.76744564063847e-05, 7.267692853929475e-05, 7.027364335954189e-05, 6.47764973109588e-05, 6.0270434914855286e-05, 5.65150985494256e-05, 5.570825669565238e-05, 5.155041435500607e-05, 4.8259222239721566e-05, 4.533546598395333e-05, 4.6176137402653694e-05, 4.259295019437559e-05, 3.9549839129904285e-05, 3.6808040022151545e-05, 4.02414589188993e-05, 3.7234472983982414e-05, 3.413966260268353e-05, 3.182896034559235e-05, 2.7707379558705725e-05, 2.687602500373032e-05, 2.633645635796711e-05, 2.5926412490662187e-05, 2.0671755919465795e-05, 2.0226119886501692e-05, 2.0108560420339927e-05, 1.99906098714564e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.0002512736391508952, "first_to_last_ratio": 5.815826874804025, "imbalance_abs_log_ratio": 1.7605829727324667, "last_quartile_mean": 4.320514426581212e-05, "mean": 0.00013164091467388062, "normalized": [ 2.22465467951502, 2.0950685860525193, 2.148742715865945, 2.011602148255569, 1.9594409030366386, 1.7609558829295935, 1.610634891401338, 1.4591459078647937, 1.4125272633741999, 1.2911637629028558, 1.1922164013125294, 1.0885645721953474, 1.0786559850616189, 0.9886788153600501, 0.9103044070794555, 0.8363940916690796, 0.853516847878044, 0.7731887115305843, 0.7048080081053143, 0.6396644863918306, 0.6683488269421659, 0.6076388883701213, 0.5570727160911497, 0.5013742768274315, 0.394539708177536, 0.3676631795999013, 0.3470701351915981, 0.3326099884380236, 0.30382715452412723, 0.2945209687200569, 0.29298846283259594, 0.29241662650296524 ], "peak_layer": 1, "peak_normalized": 2.22465467951502, "population_cv": 0.6145561150438986, "rest_mean": 0.0001427135030098725, "spike_contrast": 0.503448770748442, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 7.184893765952439e-05, "top_five_layers": [ 1, 3, 2, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00029285557684488595, 0.0002757967449724674, 0.0002828624565154314, 0.0002648091467563063, 0.00025794259272515774, 0.00023181384312920272, 0.00021202545030973852, 0.00019208330195397139, 0.00018594638095237315, 0.00016996997874230146, 0.0001569444575579837, 0.0001432996359653771, 0.0001419952604919672, 0.00013015058357268572, 0.00011983330477960408, 0.00011010368325514719, 0.00011235773854423314, 0.00010178326920140535, 9.278157085645944e-05, 8.420601807301864e-05, 8.798205089988187e-05, 7.99901390564628e-05, 7.333356188610196e-05, 6.600136839551851e-05, 5.193756805965677e-05, 4.839951725443825e-05, 4.568863005260937e-05, 4.378508310765028e-05, 3.9996084524318576e-05, 3.877100971294567e-05, 3.8569269236177206e-05, 3.8493992178700864e-05 ] } } }, "mlp_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "b9781b2e1c55e42bdc32f1e48af8d06b6449190e50665e2511e36ff3b4b6359c", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.017611408606171608, 0.014558257535099983, 0.01080919150263071, 0.010327119380235672, 0.008616750128567219, 0.007770510856062174, 0.007606723811477423, 0.008487755432724953, 0.00865182839334011, 0.009020541794598103, 0.008682658895850182, 0.008706990629434586, 0.009003226645290852, 0.009712434373795986, 0.00981286820024252, 0.011337134055793285, 0.010840619914233685, 0.010956712998449802, 0.012231985107064247, 0.014082341454923153, 0.014436324127018452, 0.016183899715542793, 0.017385432496666908, 0.02254617027938366, 0.022119678556919098, 0.022372635081410408, 0.024847008287906647, 0.026897024363279343, 0.026162855327129364, 0.024674534797668457, 0.030845578759908676, 0.029862264171242714 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 3.2954689231701195e-05, "first_to_last_ratio": 5.105739778813491, "imbalance_abs_log_ratio": 1.63036535371595, "last_quartile_mean": 6.454439642311627e-06, "mean": 1.73502516958024e-05, "normalized": [ 2.118076439435723, 2.157395185923538, 2.1393573631676532, 2.0293387218518246, 1.9547446735792444, 1.7470380730869022, 1.5859544826144767, 1.463120381926734, 1.4022787424481573, 1.2651844959124576, 1.1596399794189511, 1.0756457189909705, 1.0548334272291358, 0.9617847148778298, 0.8861896603299768, 0.8166507566511427, 0.822632314310604, 0.7462779082282098, 0.6809665471807215, 0.6235405742761265, 0.6619400891948598, 0.6046031773324564, 0.5544108586463486, 0.5123283891876923, 0.43840604673891487, 0.41933885865900805, 0.40328491124319266, 0.3940604884962617, 0.3330148316096749, 0.3301665032697355, 0.32877203485044354, 0.3290236493310315 ], "peak_layer": 2, "peak_normalized": 2.157395185923538, "population_cv": 0.5996242881005125, "rest_mean": 1.878216889289669e-05, "spike_contrast": 0.5120760486362497, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.617898831493221e-06, "top_five_layers": [ 2, 3, 1, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 3.6749159335158765e-05, 3.7431349483085796e-05, 3.7118388718226925e-05, 3.5209537600167096e-05, 3.391531208762899e-05, 3.031155029020738e-05, 2.751670945144724e-05, 2.538550688768737e-05, 2.4329889129148796e-05, 2.195126944570802e-05, 2.0120045519433916e-05, 1.8662723960005678e-05, 1.830162545957137e-05, 1.6687206880305894e-05, 1.5375613656942733e-05, 1.4169096175464801e-05, 1.4272877706389409e-05, 1.2948109542776365e-05, 1.1814940990007017e-05, 1.0818585906235967e-05, 1.1484827155072708e-05, 1.049001730279997e-05, 9.619167940400075e-06, 8.88902650331147e-06, 7.606455255881883e-06, 7.2756347435642965e-06, 6.997094715188723e-06, 6.8370486587809864e-06, 5.777891146863112e-06, 5.728471933252877e-06, 5.704277555196313e-06, 5.708643129764823e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.0001289480715058744, "first_to_last_ratio": 4.956339714424028, "imbalance_abs_log_ratio": 1.6006675075081214, "last_quartile_mean": 2.6016794436145574e-05, "mean": 6.873570305288013e-05, "normalized": [ 2.073004605358973, 2.134365700374101, 2.1192095069414485, 2.0040524994244264, 1.9289936690931089, 1.7292296834320953, 1.569549055379467, 1.4495827774599748, 1.392480193841659, 1.2691570699511074, 1.1657818381111424, 1.084357575904709, 1.0620446910222163, 0.9734770749855297, 0.8984711717160666, 0.8270404385478125, 0.8348284439914535, 0.7567459964705125, 0.691239653970941, 0.6326487613732772, 0.6743242179293075, 0.6159548780125667, 0.5640484616188776, 0.521373530426787, 0.4457155128282148, 0.4257578292711852, 0.4106916383072167, 0.4012558017197981, 0.3390531928847147, 0.336176289510552, 0.3343946322908752, 0.33499360784988247 ], "peak_layer": 2, "peak_normalized": 2.134365700374101, "population_cv": 0.5868619217766903, "rest_mean": 7.428186829740837e-05, "spike_contrast": 0.5221517931823592, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.878641073242761e-05, "top_five_layers": [ 2, 3, 1, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0001424894289812073, 0.0001467071269871667, 0.00014566535537596792, 0.0001377499575028196, 0.00013259073602966964, 0.0001188598180306144, 0.00010788405779749155, 9.963809134205803e-05, 9.571310511091724e-05, 8.723640348762274e-05, 8.013083424884826e-05, 7.4534080340527e-05, 7.300038851099089e-05, 6.691263115499169e-05, 6.175704766064882e-05, 5.68472059967462e-05, 5.7382520026294515e-05, 5.2015468099853024e-05, 4.751284359372221e-05, 4.3485557398526e-05, 4.6350149204954505e-05, 4.233809158904478e-05, 3.877026756526902e-05, 3.5836976167047396e-05, 3.0636569135822356e-05, 2.926476372522302e-05, 2.8229178496985696e-05, 2.7580599635257386e-05, 2.330505958525464e-05, 2.3107313609216362e-05, 2.2984850147622637e-05, 2.3026021153782494e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.11435244046151638, "first_to_last_ratio": 4.956339696350975, "imbalance_abs_log_ratio": 1.6006675038616696, "last_quartile_mean": 0.023071953793987632, "mean": 0.0609555093687959, "normalized": [ 2.073004695896629, 2.1343657739438058, 2.1192095214175937, 2.004052535226019, 1.9289935285682593, 1.7292296759967316, 1.5695490131514764, 1.4495827726824428, 1.3924801813690295, 1.269157021277261, 1.165781889125704, 1.0843576426531487, 1.0620447122638885, 0.9734770498631476, 0.8984712131746528, 0.8270404152126933, 0.8348284100671275, 0.7567460489501358, 0.691239607862424, 0.6326487463288526, 0.6743241627324319, 0.6159549038523712, 0.5640484241789988, 0.5213735345830548, 0.44571554071981845, 0.4257578260668156, 0.4106916263563641, 0.40125582081652367, 0.3390531599397356, 0.3361762979130643, 0.33439464558088794, 0.33499360222891106 ], "peak_layer": 2, "peak_normalized": 2.1343657739438058, "population_cv": 0.5868619269179132, "rest_mean": 0.06587390429167836, "spike_contrast": 0.522151786129607, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.03439617678523064, "top_five_layers": [ 2, 3, 1, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.12636105716228485, 0.13010135293006897, 0.1291774958372116, 0.12215804308652878, 0.1175827831029892, 0.10540607571601868, 0.09567265957593918, 0.08836005628108978, 0.08487933874130249, 0.07736211270093918, 0.07106082886457443, 0.06609757244586945, 0.0647374764084816, 0.05933878943324089, 0.05476677045226097, 0.05041266977787018, 0.05088739097118378, 0.046127840876579285, 0.04213486239314079, 0.038563426584005356, 0.041103772819042206, 0.03754584491252899, 0.03438185900449753, 0.03178058937191963, 0.027168817818164825, 0.025952285155653954, 0.025033917278051376, 0.0244587529450655, 0.02066715806722641, 0.020491797477006912, 0.020383195951581, 0.020419705659151077 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 7.461751238224679e-06, "first_to_last_ratio": 4.607278415178689, "imbalance_abs_log_ratio": 1.527637317203245, "last_quartile_mean": 1.6195572669630565e-06, "mean": 3.872976176211296e-06, "normalized": [ 3.025668897657687, 2.420035163018765, 2.225735559561749, 1.8932958237601154, 1.6740889922677953, 1.49074225689019, 1.302073782365782, 1.3813151404849735, 1.2950916709377387, 1.0715944778616184, 1.0585199120860045, 1.1104678074006644, 1.039542517960571, 0.9298369507481301, 0.7931357450923319, 0.7672068262710003, 0.8115868243354613, 0.7239540167999846, 0.6595190440563047, 0.6140523657384757, 0.6490685381015874, 0.6059744156497426, 0.5735242411534173, 0.5386197309103437, 0.45666220052738465, 0.4487521252190139, 0.4444189656933061, 0.44116761367860935, 0.38204877349263455, 0.3857109035588071, 0.3910260087310755, 0.39556270798873516 ], "peak_layer": 1, "peak_normalized": 3.025668897657687, "population_cv": 0.6453832806768665, "rest_mean": 4.185131009156369e-06, "spike_contrast": 0.5226455452702347, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.187340078307898e-06, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 1.1718343557731714e-05, 9.372738531965297e-06, 8.620220796728972e-06, 7.332689619943267e-06, 6.483706783910748e-06, 5.773609245807165e-06, 5.042900738772005e-06, 5.349800630938262e-06, 5.015859187551541e-06, 4.150259883317631e-06, 4.09962240155437e-06, 4.300815362512367e-06, 4.026123406219995e-06, 3.601236358008464e-06, 3.0717958452441962e-06, 2.971373760374263e-06, 3.1432564355782233e-06, 2.8038566597388126e-06, 2.554301545387716e-06, 2.3782101834513014e-06, 2.5138269847957417e-06, 2.3469244752050145e-06, 2.221245722466847e-06, 2.0860613858530996e-06, 1.7686418232187862e-06, 1.7380062899974291e-06, 1.7212240663866396e-06, 1.7086316574932425e-06, 1.4796657978877192e-06, 1.4938491403881926e-06, 1.5144344160944456e-06, 1.5320049442379968e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 0.00010837215813808143, "first_to_last_ratio": 4.663749544487092, "imbalance_abs_log_ratio": 1.5398197478822726, "last_quartile_mean": 2.3237130790221272e-05, "mean": 5.872993017419503e-05, "normalized": [ 2.116908005707516, 2.1257566114370654, 2.0713368933145166, 1.958807069139986, 1.853064595344867, 1.6774795353818415, 1.534246942340664, 1.4245031207093415, 1.3659761301608329, 1.2527187093598229, 1.1583387848911404, 1.08159561433793, 1.0590461894491214, 0.974994469979534, 0.9039318927032919, 0.837094157953219, 0.8447810620709018, 0.7717232000223526, 0.7087798354672495, 0.6523243210626272, 0.6962576906319006, 0.6381512392879863, 0.585489683522781, 0.5414078956394395, 0.4665886811070132, 0.44848929689996414, 0.43561388426614667, 0.42649458236277493, 0.3502420718536938, 0.3476010806933041, 0.3452465505067908, 0.3450102023943839 ], "peak_layer": 2, "peak_normalized": 2.1257566114370654, "population_cv": 0.5682342302969126, "rest_mean": 6.323713649989158e-05, "spike_contrast": 0.5438420826580702, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.4391016015433706e-05, "top_five_layers": [ 2, 1, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00012432585936039686, 0.0001248455373570323, 0.00012164947111159563, 0.000115040602395311, 0.00010883035429287702, 9.851825598161668e-05, 9.010621579363942e-05, 8.366096881218255e-05, 8.022368274396285e-05, 7.357208232861012e-05, 6.802915595471859e-05, 6.352203490678221e-05, 6.219770875759423e-05, 5.7261357142124325e-05, 5.308785694069229e-05, 4.916248144581914e-05, 4.961393278790638e-05, 4.532324965111911e-05, 4.162659024586901e-05, 3.831096182693727e-05, 4.089116555405781e-05, 3.747857772395946e-05, 3.4385768231004477e-05, 3.179684790666215e-05, 2.7402720661484636e-05, 2.633974509080872e-05, 2.558357300586067e-05, 2.5047997041838244e-05, 2.0569692424032837e-05, 2.0414587197592482e-05, 2.0276305804145522e-05, 2.0262425096007064e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 9.030398450704524e-05, "first_to_last_ratio": 3.89627084299362, "imbalance_abs_log_ratio": 1.360019901586191, "last_quartile_mean": 2.3177029561338713e-05, "mean": 5.084823260403937e-05, "normalized": [ 2.109723573221063, 2.0821310668378783, 1.9942546197542355, 1.874953814933449, 1.728017146958865, 1.5814957309589324, 1.4629098120324804, 1.3741245898657877, 1.3101737824863438, 1.2101890628861518, 1.1282931314057325, 1.0632895987411368, 1.0440174426577644, 0.9649796238381606, 0.9017142254727002, 0.8410904420982939, 0.8516872374927745, 0.7851358108806284, 0.7291665843144397, 0.6770579705243002, 0.7378064582735888, 0.6786248956615917, 0.6313013382664657, 0.5913982993163636, 0.5312183147012203, 0.5201489591253866, 0.5114845807208003, 0.5059300847390553, 0.39756423665978435, 0.3954241221367383, 0.3931178424325881, 0.39157560060529917 ], "peak_layer": 1, "peak_normalized": 2.109723573221063, "population_cv": 0.5213662772063792, "rest_mean": 5.429395497668776e-05, "spike_contrast": 0.5938291252788923, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.224133179173805e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00010727571498136967, 0.00010587268479866907, 0.00010140432277694345, 9.5338087703567e-05, 8.786661783233285e-05, 8.041626279009506e-05, 7.438637840095907e-05, 6.987180677242577e-05, 6.662002124357969e-05, 6.153597496449947e-05, 5.7371711591258645e-05, 5.406639684224501e-05, 5.308644176693633e-05, 4.90675083710812e-05, 4.585057467920706e-05, 4.276796244084835e-05, 4.330679075792432e-05, 3.9922768337419257e-05, 3.7076832086313516e-05, 3.4427201171638444e-05, 3.751615440705791e-05, 3.450687654549256e-05, 3.210055729141459e-05, 3.007155828527175e-05, 2.7011512429453433e-05, 2.6448655262356624e-05, 2.60080869338708e-05, 2.572565063019283e-05, 2.021543878072407e-05, 2.010661773965694e-05, 1.998934749281034e-05, 1.991092722164467e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00023694485207670368, "first_to_last_ratio": 5.643445556811249, "imbalance_abs_log_ratio": 1.7304947933060113, "last_quartile_mean": 4.198584883852163e-05, "mean": 0.00012326974319876172, "normalized": [ 2.0534243627746784, 2.17548929765909, 2.1923221930833754, 2.045443753327701, 2.026125682616779, 1.7904266625829455, 1.6133287652163053, 1.4807635751829418, 1.4348095534638061, 1.3014197885977827, 1.1747669830911103, 1.0855875327354585, 1.0626766295217445, 0.9759316522493466, 0.8962992674239908, 0.8231145619826911, 0.8258276905981559, 0.7455975627999764, 0.6776587351559191, 0.6226196341065754, 0.6492096639107944, 0.5955964221479619, 0.5398643713108281, 0.4868843463557821, 0.3972339380481051, 0.3723774041823533, 0.3563689360290182, 0.3460491488507791, 0.3146328182140966, 0.31212742697325935, 0.31138985452451795, 0.31463178528213015 ], "peak_layer": 3, "peak_normalized": 2.1923221930833754, "population_cv": 0.61857583014248, "rest_mean": 0.00013391299553741528, "spike_contrast": 0.4913352905442922, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.579618057003244e-05, "top_five_layers": [ 3, 2, 1, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0002531250938773155, 0.0002681720070540905, 0.0002702469937503338, 0.000252141326200217, 0.00024975999258458614, 0.0002207054349128157, 0.00019887462258338928, 0.00018253334565088153, 0.00017686860519461334, 0.00016042568313423544, 0.00014481322432402521, 0.0001338200963800773, 0.0001309958752244711, 0.00012030284415232018, 0.00011048658052459359, 0.00010146512067876756, 0.00010179956734646112, 9.19096200959757e-05, 8.353481825906783e-05, 7.675016240682453e-05, 8.002790855243802e-05, 7.341901800828055e-05, 6.654894241364673e-05, 6.001810834277421e-05, 4.896692553302273e-05, 4.590286698658019e-05, 4.3929507228313014e-05, 4.2657389712985605e-05, 3.878470670315437e-05, 3.8475867768283933e-05, 3.83849474019371e-05, 3.878457937389612e-05 ] } } }, "post_attention_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "3706b7098979776b3ef475b6da00314debf1299b08b4c6a8322412051a6af174", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.010549778118729591, 0.02910016104578972, 0.04255657270550728, 0.05177471786737442, 0.01913420483469963, 0.02246284671127796, 0.029843268916010857, 0.036163609474897385, 0.019129201769828796, 0.0314205139875412, 0.03302976116538048, 0.03871368616819382, 0.018231205642223358, 0.02580643631517887, 0.0349617674946785, 0.04659247398376465, 0.03611915931105614, 0.03664228320121765, 0.04177439957857132, 0.04776441678404808, 0.013802599161863327, 0.02693362347781658, 0.03932841867208481, 0.0514267161488533, 0.015529165975749493, 0.03558400645852089, 0.05707564577460289, 0.08332383632659912, 0.009738311171531677, 0.03187238425016403, 0.05786185711622238, 0.09122034162282944 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 3.478662188172166e-05, "first_to_last_ratio": 5.263389233448399, "imbalance_abs_log_ratio": 1.6607751602108825, "last_quartile_mean": 6.609167655824422e-06, "mean": 1.839926902391653e-05, "normalized": [ 2.3257405363021224, 2.093986543353821, 2.0972432569142194, 1.9748429471145472, 1.9001738268978305, 1.7176742899564605, 1.5731126681977952, 1.4424450196971237, 1.3881676796557327, 1.2578123363623133, 1.1637254248597446, 1.0739089583500412, 1.0625597930502229, 0.9721654778743629, 0.897852656475923, 0.8303134671066331, 0.8465754829823255, 0.7713090877071682, 0.7043469066766334, 0.642452591912055, 0.6810739404273755, 0.621476487606478, 0.5661443389519891, 0.5212309648706807, 0.4305042754919776, 0.4113531398630616, 0.3923510140548581, 0.378496313745133, 0.3251027262595914, 0.314172044005768, 0.31172491175027217, 0.3099608915257388 ], "peak_layer": 1, "peak_normalized": 2.3257405363021224, "population_cv": 0.5996654929960216, "rest_mean": 1.9884546603592485e-05, "spike_contrast": 0.5219515586938898, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.037877009366639e-05, "top_five_layers": [ 1, 3, 2, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 4.2791925807250664e-05, 3.852782174362801e-05, 3.858774289255962e-05, 3.633566666394472e-05, 3.4961809433298185e-05, 3.160395135637373e-05, 2.8944123187102377e-05, 2.653993396961596e-05, 2.554127058829181e-05, 2.314282755833119e-05, 2.1411697161966003e-05, 1.9759139831876382e-05, 1.9550323486328125e-05, 1.7887134163174778e-05, 1.6519832570338622e-05, 1.5277160855475813e-05, 1.5576370060443878e-05, 1.419152340531582e-05, 1.2959468222106807e-05, 1.1820658073702361e-05, 1.2531262655102182e-05, 1.1434713087510318e-05, 1.0416641998745035e-05, 9.59026874625124e-06, 7.920963980723172e-06, 7.568597084173234e-06, 7.2189718594017904e-06, 6.964055501157418e-06, 5.9816525208589155e-06, 5.780535957455868e-06, 5.735510512749897e-06, 5.703053830075078e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.00013640988436236512, "first_to_last_ratio": 5.122011664762131, "imbalance_abs_log_ratio": 1.6335472651589908, "last_quartile_mean": 2.6632091703504557e-05, "mean": 7.298115508547198e-05, "normalized": [ 2.2789966397783896, 2.0685142340171683, 2.076140409583827, 1.9537250082825501, 1.8792316671609142, 1.7055529809049683, 1.5594088420998298, 1.4313187223467239, 1.382107687555985, 1.2644854892365605, 1.1686221274742359, 1.0808639824268664, 1.0704039370079022, 0.9820006566284564, 0.9088834247617501, 0.8405321787627472, 0.856406336273156, 0.7807790484250425, 0.7130462191259114, 0.650909996157966, 0.6919497653761542, 0.6322822243882543, 0.5753541906206389, 0.5291452134901508, 0.4371545081034775, 0.4171814083388854, 0.3982526780954131, 0.3849724710776064, 0.3303152718064163, 0.31952201613190695, 0.31683873767491905, 0.31510192688522626 ], "peak_layer": 1, "peak_normalized": 2.2789966397783896, "population_cv": 0.5877439115216546, "rest_mean": 7.874967775057519e-05, "spike_contrast": 0.5311911602534674, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.1831132693914694e-05, "top_five_layers": [ 1, 3, 2, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00016632380720693618, 0.00015096255810931325, 0.0001515191252110526, 0.00014258510782383382, 0.00013714849774260074, 0.00012447322660591453, 0.00011380745854694396, 0.00010445929365232587, 0.0001008678154903464, 9.228361159330234e-05, 8.528739272151142e-05, 7.8882701927796e-05, 7.81193157308735e-05, 7.16675422154367e-05, 6.633136217715219e-05, 6.134300929261371e-05, 6.250152364373207e-05, 5.6982156820595264e-05, 5.203893670113757e-05, 4.750416337628849e-05, 5.049929313827306e-05, 4.6144687075866386e-05, 4.199001341476105e-05, 3.8617628888459876e-05, 3.190404095221311e-05, 3.044638106075581e-05, 2.9064940463285893e-05, 2.809573561535217e-05, 2.4106790078803897e-05, 2.331908581254538e-05, 2.312325705133844e-05, 2.2996502593741752e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.12096964661031961, "first_to_last_ratio": 5.122011546850009, "imbalance_abs_log_ratio": 1.6335472421383241, "last_quartile_mean": 0.023617605213075876, "mean": 0.06472041795495898, "normalized": [ 2.278996550367558, 2.068514172728323, 2.0761403775307596, 1.9537250896945753, 1.879231622178916, 1.7055529409943957, 1.5594087663016845, 1.431318663899547, 1.382107735687561, 1.2644854472317082, 1.1686221211602732, 1.0808640120306183, 1.0704040262086971, 0.9820006724164564, 0.9088834582355364, 0.8405322479178425, 0.8564063602524741, 0.7807790810183837, 0.713046234965646, 0.6509099797949671, 0.6919497851652032, 0.6322822095960979, 0.575354189755429, 0.529145232117261, 0.4371544964362311, 0.41718141126177893, 0.3982526899729607, 0.38497247844984545, 0.33031527661149607, 0.3195220199067584, 0.3168387288221828, 0.3151019212888319 ], "peak_layer": 1, "peak_normalized": 2.278996550367558, "population_cv": 0.5877438959595169, "rest_mean": 0.06983600149827975, "spike_contrast": 0.5311911625115104, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.037096266821026805, "top_five_layers": [ 1, 3, 2, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.14749760925769806, 0.13387510180473328, 0.1343686729669571, 0.12644590437412262, 0.121624656021595, 0.11038409918546677, 0.10092558711767197, 0.09263554215431213, 0.08945059031248093, 0.08183802664279938, 0.0756337121129036, 0.06995397061109543, 0.06927699595689774, 0.06355549395084381, 0.05882331728935242, 0.05439959838986397, 0.05542697757482529, 0.050532348453998566, 0.04614865034818649, 0.04212716594338417, 0.04478327929973602, 0.040921568870544434, 0.03723716363310814, 0.034246500581502914, 0.0282928217202425, 0.027000155299901962, 0.025775080546736717, 0.024915579706430435, 0.02137814275920391, 0.020679598674178123, 0.020505934953689575, 0.020393528044223785 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 7.71177167280257e-06, "first_to_last_ratio": 4.678295917137282, "imbalance_abs_log_ratio": 1.542933923351862, "last_quartile_mean": 1.6484146811990286e-06, "mean": 4.007248069370917e-06, "normalized": [ 3.3590280906213086, 2.3726088061078587, 2.138143013037793, 1.8393897753025719, 1.595536022354422, 1.4487646483149947, 1.2955498112963995, 1.3466260007113127, 1.2739769202114613, 1.0659628023122865, 1.0478341793654378, 1.0914579443825887, 1.0411457104586188, 0.9155146938805809, 0.8066044523853629, 0.7701707432135287, 0.8215762550934921, 0.7364785130598478, 0.6730090976811546, 0.6266208737012697, 0.6693583504854638, 0.6236588439557476, 0.5897891291930596, 0.5603290672032583, 0.4588325813842428, 0.4472228580494504, 0.4388965999812878, 0.43180771263272905, 0.37877595830431615, 0.3751711426274845, 0.3796103005144005, 0.3805491021762693 ], "peak_layer": 1, "peak_normalized": 3.3590280906213086, "population_cv": 0.6648044266247514, "rest_mean": 4.318630839474348e-06, "spike_contrast": 0.5385459413556807, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.325781110812386e-06, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 1.3460458831104916e-05, 9.507632057648152e-06, 8.568069461034611e-06, 7.370891125901835e-06, 6.393708645191509e-06, 5.805559339933097e-06, 5.191589480091352e-06, 5.396264441515086e-06, 5.105141553940484e-06, 4.271577381587122e-06, 4.198931492283009e-06, 4.373742740426678e-06, 4.172129138169112e-06, 3.6686944895336637e-06, 3.232264134567231e-06, 3.086265223828377e-06, 3.292259862064384e-06, 2.9512520995922387e-06, 2.696914407351869e-06, 2.5110252863669302e-06, 2.682284957700176e-06, 2.499155698387767e-06, 2.3634313492948422e-06, 2.2453775727626635e-06, 1.8386559759164811e-06, 1.7921329344972037e-06, 1.7587675529284752e-06, 1.730360622786975e-06, 1.5178492276390898e-06, 1.5034038369776681e-06, 1.5211926438496448e-06, 1.5249546549966908e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 0.00011456928586994763, "first_to_last_ratio": 4.836938296613537, "imbalance_abs_log_ratio": 1.5762819372189676, "last_quartile_mean": 2.3686323629590333e-05, "mean": 6.211567796299278e-05, "normalized": [ 2.3488758065274955, 2.0713117417995783, 2.0294413849686572, 1.9098281260734524, 1.808891788235116, 1.6555687759458058, 1.5234656987314679, 1.4082197237907075, 1.3570554765296423, 1.2477441770945124, 1.1584743113969003, 1.07706541625456, 1.062967326057557, 0.9794184805192363, 0.9103158522005721, 0.8467437815449563, 0.8606016851807378, 0.7903183885158978, 0.726053695018778, 0.6672539632450412, 0.7127881461452974, 0.6531207323660743, 0.5952249360680494, 0.548642506624817, 0.4562346365956528, 0.4371660140704956, 0.4212530247856756, 0.40928721260242446, 0.3409968025795472, 0.33116514257975266, 0.32832896742236684, 0.3261762785291743 ], "peak_layer": 1, "peak_normalized": 2.3488758065274955, "population_cv": 0.5738573169932581, "rest_mean": 6.679503382690234e-05, "spike_contrast": 0.5516451476522887, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.6847156297881153e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00014590201317332685, 0.00012866093311458826, 0.00012606012751348317, 0.00011863026884384453, 0.0001123605397879146, 0.00010283677693223581, 9.463110473006964e-05, 8.747252286411822e-05, 8.429442095803097e-05, 7.750447548460215e-05, 7.195941725512967e-05, 6.690264854114503e-05, 6.602693611057475e-05, 6.08372429269366e-05, 5.654488631989807e-05, 5.259606405161321e-05, 5.34568571310956e-05, 4.9091162509284914e-05, 4.509931750362739e-05, 4.1446932300459594e-05, 4.4275318941799924e-05, 4.0569037082605064e-05, 3.697280044434592e-05, 3.4079301258316264e-05, 2.833932376233861e-05, 2.7154863346368074e-05, 2.6166417228523642e-05, 2.5423152692383155e-05, 2.1181247575441375e-05, 2.0570547349052504e-05, 2.0394376406329684e-05, 2.0260660676285625e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 9.518698971078265e-05, "first_to_last_ratio": 4.0537764308209505, "imbalance_abs_log_ratio": 1.399648898709643, "last_quartile_mean": 2.3481065454689087e-05, "mean": 5.345527347344614e-05, "normalized": [ 2.3490757134047047, 2.03180856162592, 1.9640798219543283, 1.8355777168881, 1.687288592851433, 1.564711486655573, 1.4531995506450504, 1.3597364706007256, 1.3148292531115187, 1.211773745536977, 1.127520479381137, 1.0571877619151735, 1.041978223119568, 0.9643430432278, 0.9029106186893467, 0.848047824187126, 0.863550005931762, 0.7965670293846996, 0.7398915500109849, 0.6885201220036962, 0.7527468103098911, 0.696422774701012, 0.6386525384719246, 0.5954551044265526, 0.5183504589787767, 0.502904662044204, 0.4927615213178117, 0.48500410119101467, 0.3866832502775355, 0.3782793274311306, 0.37617794688759315, 0.3739639328369289 ], "peak_layer": 1, "peak_normalized": 2.3490757134047047, "population_cv": 0.5319468462674285, "rest_mean": 5.701573583657456e-05, "spike_contrast": 0.6003391206010799, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.4228776712552644e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00012557048466987908, 0.00010861088230740279, 0.00010499042400624603, 9.812130883801728e-05, 9.019447315949947e-05, 8.364208042621613e-05, 7.768117939122021e-05, 7.268508488778025e-05, 7.028455729596317e-05, 6.477569695562124e-05, 6.027191557222977e-05, 5.651226092595607e-05, 5.5699230870231986e-05, 5.154922109795734e-05, 4.826533404411748e-05, 4.5332628360483795e-05, 4.6161301725078374e-05, 4.2580708395689726e-05, 3.955110514652915e-05, 3.680503141367808e-05, 4.0238286601379514e-05, 3.722746987477876e-05, 3.413934609852731e-05, 3.1830215448280796e-05, 2.7708565539796837e-05, 2.6882906240643933e-05, 2.6340701879234985e-05, 2.5926026864908636e-05, 2.0670258891186677e-05, 2.022102489718236e-05, 2.010869502555579e-05, 1.9990344299003482e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.0002512551309337141, "first_to_last_ratio": 5.815353323084221, "imbalance_abs_log_ratio": 1.7605015447583472, "last_quartile_mean": 4.320547986935708e-05, "mean": 0.0001316397593882357, "normalized": [ 2.224752247026743, 2.0950471769673564, 2.148762457826491, 2.0117752264198114, 1.9594249362752976, 1.7610216345575365, 1.6099537081187654, 1.4585175613957049, 1.4124641586885858, 1.2915649811078755, 1.192264449118915, 1.0889144888939652, 1.0788791319896314, 0.988941963176937, 0.9104012729618977, 0.8363983367426334, 0.8532666616213362, 0.773042726053889, 0.7048431559693451, 0.6396986756467042, 0.668255590231208, 0.6075232312555597, 0.5571337058341255, 0.5014728599784464, 0.3945136003262428, 0.3676589998419824, 0.34703606615034643, 0.33261343254293874, 0.3038165004585062, 0.2945782724854268, 0.2930231193739135, 0.29243967096188167 ], "peak_layer": 1, "peak_normalized": 2.224752247026743, "population_cv": 0.6145445602222892, "rest_mean": 0.0001427126183073226, "spike_contrast": 0.503433558134641, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 7.184632122516632e-05, "top_five_layers": [ 1, 3, 2, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00029286585049703717, 0.0002757915062829852, 0.00028286257293075323, 0.0002648296067491174, 0.00025793822715058923, 0.00023182046425063163, 0.0002119339187629521, 0.00019199890084564686, 0.00018593644199427217, 0.00017002130334731191, 0.00015694940520916134, 0.00014334444131236523, 0.00014202338934410363, 0.00013018408208154142, 0.00011984500451944768, 0.0001101032758015208, 0.00011232381802983582, 0.00010176315845455974, 9.278538345824927e-05, 8.420977974310517e-05, 8.796900510787964e-05, 7.997421198524535e-05, 7.334094698308036e-05, 6.601376662729308e-05, 5.193367542233318e-05, 4.839854227611795e-05, 4.5683744247071445e-05, 4.3785152229247615e-05, 3.999433101853356e-05, 3.877821291098371e-05, 3.857349292957224e-05, 3.849668792099692e-05 ] } } }, "post_mlp_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "2674fdc2460a10ee1ce2d4e7e0c398dced26688787f8d47d6c42ecf46862e67d", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.021530626341700554, 0.03857670724391937, 0.04954109713435173, 0.057694368064403534, 0.02140580676496029, 0.025963041931390762, 0.03339661285281181, 0.040483180433511734, 0.02151828072965145, 0.035047177225351334, 0.03725159168243408, 0.043239567428827286, 0.02124657854437828, 0.03028978779911995, 0.03979094699025154, 0.05222286656498909, 0.03916332125663757, 0.04088718071579933, 0.04756740853190422, 0.05542457476258278, 0.020541559904813766, 0.03680417314171791, 0.05126549303531647, 0.06891486048698425, 0.02905990369617939, 0.05358703061938286, 0.07880406081676483, 0.10752817988395691, 0.029148343950510025, 0.054091889411211014, 0.08628532290458679, 0.11884573101997375 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 3.2954461630652077e-05, "first_to_last_ratio": 5.105695837816317, "imbalance_abs_log_ratio": 1.6303567474829335, "last_quartile_mean": 6.454450613091467e-06, "mean": 1.7350209816413553e-05, "normalized": [ 2.118074842246104, 2.1574284903940977, 2.139357285099935, 2.0292905713578766, 1.9547395369559644, 1.7470193301554893, 1.5858965602051887, 1.4631504379843852, 1.4022879982430452, 1.2651865013786623, 1.1596469721061315, 1.075647057275218, 1.0548378604659292, 0.9617861976864589, 0.8861851944905509, 0.81666640942737, 0.8226291628131417, 0.7462778224571112, 0.6809633158339518, 0.623540506764385, 0.6619488684789228, 0.6046004431197245, 0.5544183299835491, 0.5123407388295048, 0.4384090182726579, 0.4193402639944847, 0.4032847838635732, 0.3940637985584808, 0.33301498018274, 0.33016832240151506, 0.32877371956677426, 0.3290246794070765 ], "peak_layer": 2, "peak_normalized": 2.1574284903940977, "population_cv": 0.5996208655691896, "rest_mean": 1.8782105026157632e-05, "spike_contrast": 0.5120818816847563, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.61797568379552e-06, "top_five_layers": [ 2, 3, 1, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 3.674904291983694e-05, 3.7431836972245947e-05, 3.711829776875675e-05, 3.52086171915289e-05, 3.391514110262506e-05, 3.0311151931528002e-05, 2.7515638066688552e-05, 2.5385967092006467e-05, 2.4329990992555395e-05, 2.1951251255813986e-05, 2.0120118279010057e-05, 1.8662702132132836e-05, 1.8301658201380633e-05, 1.6687192328390665e-05, 1.537549906061031e-05, 1.4169333553581964e-05, 1.4272788575908635e-05, 1.2948076800967101e-05, 1.1814856406999752e-05, 1.0818558621394914e-05, 1.148495175584685e-05, 1.0489944543223828e-05, 9.619274351280183e-06, 8.889219316188246e-06, 7.606488452438498e-06, 7.27564156477456e-06, 6.997075615799986e-06, 6.837089586042566e-06, 5.77787977817934e-06, 5.728489668399561e-06, 5.704293016606243e-06, 5.708647222490981e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 0.00012894753399450565, "first_to_last_ratio": 4.956313899688395, "imbalance_abs_log_ratio": 1.600662299067145, "last_quartile_mean": 2.601682149361295e-05, "mean": 6.873563796716553e-05, "normalized": [ 2.0729820101052394, 2.1343785185339303, 2.1192055857825096, 2.004033226215171, 1.9290001532413559, 1.7292283569198776, 1.5695265126752398, 1.449584785193964, 1.392505223726082, 1.2691464160417294, 1.1657826244262586, 1.0843620958713254, 1.0620513069452608, 0.9734787377496757, 0.8984695878304712, 0.827044661933948, 0.8348303459591176, 0.7567409969045322, 0.6912396733798123, 0.6326506836056272, 0.674324856446311, 0.615953502956551, 0.5640541296457265, 0.5213814868377082, 0.4457146646251511, 0.42575966145291916, 0.4106923182898804, 0.40125618166856064, 0.3390529581992065, 0.3361781427218294, 0.33439592808042723, 0.33499466603460004 ], "peak_layer": 2, "peak_normalized": 2.1343785185339303, "population_cv": 0.5868596491376813, "rest_mean": 7.428176730981149e-05, "spike_contrast": 0.5221542367874475, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.8786539516877386e-05, "top_five_layers": [ 2, 3, 1, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00014248774095904082, 0.00014670786913484335, 0.00014566494792234153, 0.00013774850231129676, 0.00013259105617180467, 0.00011885961430380121, 0.00010788240615511313, 9.963813499780372e-05, 9.571473492542282e-05, 8.723558858036995e-05, 8.013081242097542e-05, 7.453432044712827e-05, 7.300077413674444e-05, 6.691268208669499e-05, 6.17568803136237e-05, 5.684744246536866e-05, 5.7382596423849463e-05, 5.201507519814186e-05, 4.7512799937976524e-05, 4.3485648347996175e-05, 4.6350149204954505e-05, 4.233795698382892e-05, 3.877062044921331e-05, 3.5837489122059196e-05, 3.0636481824330986e-05, 2.926486195065081e-05, 2.8229198505869135e-05, 2.7580599635257386e-05, 2.3305021386477165e-05, 2.3107419110601768e-05, 2.298491745023057e-05, 2.3026072085485794e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.11435196176171303, "first_to_last_ratio": 4.956313746493952, "imbalance_abs_log_ratio": 1.6006622681581977, "last_quartile_mean": 0.023071978008374572, "mean": 0.06095545022981241, "normalized": [ 2.072982016679514, 2.1343786009397814, 2.119205465978219, 2.004033333779568, 1.9290000448154139, 1.7292284201771326, 1.5695265788612092, 1.4495847902174488, 1.392505122730498, 1.2691463963100211, 1.1657826534776023, 1.0843621171351583, 1.0620512430067517, 0.97347872771027, 0.8984695791578813, 0.8270447011604714, 0.8348303200871255, 0.7567409772234059, 0.691239667353491, 0.6326506435390216, 0.6743248169617974, 0.6159534846579819, 0.5640541661909846, 0.5213814964456189, 0.4457146897387812, 0.4257596753388107, 0.4106923303842507, 0.4012562101149584, 0.33905296941192775, 0.3361781519455022, 0.33439594785006815, 0.3349946606193331 ], "peak_layer": 2, "peak_normalized": 2.1343786009397814, "population_cv": 0.586859647417989, "rest_mean": 0.0658738130910529, "spike_contrast": 0.5221542395241354, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.03439629077911377, "top_five_layers": [ 2, 3, 1, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.12635955214500427, 0.1301020085811615, 0.12917712330818176, 0.12215675413608551, 0.11758306622505188, 0.10540589690208435, 0.09567119926214218, 0.08836009353399277, 0.0848807767033577, 0.07736138999462128, 0.07106080651283264, 0.06609778106212616, 0.06473781168460846, 0.05933883413672447, 0.054766617715358734, 0.05041288211941719, 0.05088745802640915, 0.04612748697400093, 0.04213482514023781, 0.038563504815101624, 0.041103772819042206, 0.03754572197794914, 0.0343821756541729, 0.031781043857336044, 0.027168739587068558, 0.025952372699975967, 0.02503393590450287, 0.0244587529450655, 0.020667126402258873, 0.020491890609264374, 0.020383255556225777, 0.02041975036263466 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 7.461771360794955e-06, "first_to_last_ratio": 4.607249847489048, "imbalance_abs_log_ratio": 1.5276311166276264, "last_quartile_mean": 1.6195716767697377e-06, "mean": 3.87299716564371e-06, "normalized": [ 3.0256593103213434, 2.4199948075671043, 2.2257260804728656, 1.8933484975335824, 1.6741432062616066, 1.4907581305464577, 1.3020525186737137, 1.38123110006882, 1.2950838303767327, 1.0716197853648062, 1.0585124142900662, 1.1105323556195759, 1.0395543790453834, 0.9297870003858463, 0.7930926998517134, 0.7672014355923814, 0.811551487187536, 0.7239809147787968, 0.6594474866461876, 0.6140918356353618, 0.649091321445497, 0.6059854562252807, 0.5735486080526876, 0.5386444043869645, 0.45665673160062437, 0.44875544656608446, 0.4444235140257583, 0.44115113302107534, 0.3820499025609541, 0.38571480138297476, 0.3910437033480733, 0.3955657011641444 ], "peak_layer": 1, "peak_normalized": 3.0256593103213434, "population_cv": 0.6453809227293102, "rest_mean": 4.185142588371322e-06, "spike_contrast": 0.5226612562717605, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.1874118829146026e-06, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 1.1718369933078066e-05, 9.372633030579891e-06, 8.620230801170692e-06, 7.332933364523342e-06, 6.483951892732875e-06, 5.773702014266746e-06, 5.042845714342548e-06, 5.349504135665484e-06, 5.015856004320085e-06, 4.150380391365616e-06, 4.099615580344107e-06, 4.30108866567025e-06, 4.026191163575277e-06, 3.60106241714675e-06, 3.0716457786184037e-06, 2.9713689855270786e-06, 3.1431366096512647e-06, 2.8039760309184203e-06, 2.5540382466715528e-06, 2.3783759388606995e-06, 2.5139288482023403e-06, 2.3469799543818226e-06, 2.2213521333469544e-06, 2.086168251480558e-06, 1.7686302271613386e-06, 1.7380285726176226e-06, 1.7212510101671796e-06, 1.7085770878111362e-06, 1.4796781897530309e-06, 1.4938723325030878e-06, 1.5145111547099077e-06, 1.5320248394345981e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 0.00010837210174940992, "first_to_last_ratio": 4.663743740878985, "imbalance_abs_log_ratio": 1.5398185034733132, "last_quartile_mean": 2.3237147615873255e-05, "mean": 5.8729946545099665e-05, "normalized": [ 2.116910388942848, 2.12576419551759, 2.0713182282290723, 1.958808009785277, 1.85306829100999, 1.6774795633398605, 1.5342385858151337, 1.4245037147377415, 1.3659845454711594, 1.2527089446494675, 1.1583413114381589, 1.0816008878212584, 1.059050106445956, 0.9749946937544519, 0.9039297824077048, 0.8370924379536968, 0.8447837999101473, 0.7717212504681601, 0.708779947616627, 0.6523240772837903, 0.6962576204392495, 0.6381480261388907, 0.5854918741973255, 0.5414119569273494, 0.4665866617482466, 0.4484909063213352, 0.43561453714162146, 0.42649489708721455, 0.35024135478224894, 0.347602098795276, 0.34524713565577436, 0.34501016816737656 ], "peak_layer": 2, "peak_normalized": 2.12576419551759, "population_cv": 0.5682337234037166, "rest_mean": 6.323715206235647e-05, "spike_contrast": 0.5438422767363214, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.4391036751912904e-05, "top_five_layers": [ 2, 1, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.0001243260339833796, 0.00012484601757023484, 0.00012164840882178396, 0.00011504068970680237, 0.0001088306016754359, 9.851828508544713e-05, 9.010575013235211e-05, 8.366102701984346e-05, 8.022419933695346e-05, 7.357152935583144e-05, 6.802932330174372e-05, 6.352236232487485e-05, 6.219795614015311e-05, 5.726138624595478e-05, 5.308774780132808e-05, 4.916239413432777e-05, 4.961410741088912e-05, 4.5323147787712514e-05, 4.1626608435763046e-05, 3.8310958188958466e-05, 4.089117283001542e-05, 3.747839946299791e-05, 3.4385906474199146e-05, 3.1797095289221033e-05, 2.740260970313102e-05, 2.6339846954215318e-05, 2.558361848059576e-05, 2.5048022507689893e-05, 2.0569656044244766e-05, 2.041465268121101e-05, 2.02763458219124e-05, 2.026242873398587e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 9.029894954437623e-05, "first_to_last_ratio": 3.896020389480804, "imbalance_abs_log_ratio": 1.3599556192062257, "last_quartile_mean": 2.3177227149062674e-05, "mean": 5.08472848537167e-05, "normalized": [ 2.109310432443945, 2.0823500317454187, 1.994011612294058, 1.875066462699875, 1.727617067796283, 1.5815194849270788, 1.4631049290688447, 1.374102981258909, 1.3102874938711893, 1.2100951410093623, 1.1285247251248958, 1.0633171446519065, 1.0439918275480478, 0.9649933174114446, 0.9016610595399868, 0.8411183538760414, 0.8515897815206308, 0.7853791814165375, 0.7291252271433669, 0.6771443554330082, 0.737694287360078, 0.6786209457138218, 0.6314365240390994, 0.5913748367488056, 0.5313425485217884, 0.5201071760800584, 0.5115574693700321, 0.5058856398361036, 0.39753526518974286, 0.39544594503370456, 0.39314180453082337, 0.39154694679511176 ], "peak_layer": 1, "peak_normalized": 2.109310432443945, "population_cv": 0.5213289902477497, "rest_mean": 5.429271732282359e-05, "spike_contrast": 0.5938540399226916, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.224194952053949e-05, "top_five_layers": [ 1, 2, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00010725270840339363, 0.00010588184522930533, 0.00010139007645193487, 9.53420385485515e-05, 8.78446371643804e-05, 8.04159717517905e-05, 7.439491309924051e-05, 6.986940570641309e-05, 6.662456144113094e-05, 6.153005233500153e-05, 5.7382418162887916e-05, 5.406678974395618e-05, 5.3084149840287864e-05, 4.906729009235278e-05, 4.584701673593372e-05, 4.276858453522436e-05, 4.3301028199493885e-05, 3.993439895566553e-05, 3.707403811858967e-05, 3.4430951927788556e-05, 3.750975156435743e-05, 3.4506032534409314e-05, 3.210683280485682e-05, 3.006980477948673e-05, 2.7017325919587165e-05, 2.644603773660492e-05, 2.601130836410448e-05, 2.572291123215109e-05, 2.0213588868500665e-05, 2.0107352611375973e-05, 1.9990193322882988e-05, 1.9909099137294106e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.0002369592275499599, "first_to_last_ratio": 5.643806467094249, "imbalance_abs_log_ratio": 1.7305587433768272, "last_quartile_mean": 4.1985711050074315e-05, "mean": 0.00012327438730608264, "normalized": [ 2.0536938202390447, 2.174916037645403, 2.1925488795748866, 2.045836750341831, 2.0263612272059848, 1.7896425612539104, 1.6137064052293353, 1.480972211102483, 1.4347200865083345, 1.3015150111942828, 1.1747028946305826, 1.0857434163548976, 1.0627794297334374, 0.9758997849090596, 0.8960919751718553, 0.8230876844179131, 0.8258333502423459, 0.7456000475898096, 0.6776422362024666, 0.6227334054207839, 0.6492856625079976, 0.5954634942404196, 0.5397936868798249, 0.4867302228264425, 0.3971938000026409, 0.37230063475944564, 0.35633216718416544, 0.3460424570800059, 0.31457528168990173, 0.31217590063925915, 0.31141034981968424, 0.31466912740156466 ], "peak_layer": 3, "peak_normalized": 2.1925488795748866, "population_cv": 0.618597930641205, "rest_mean": 0.00013391951021427909, "spike_contrast": 0.49127064082412475, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.57907236018218e-05, "top_five_layers": [ 3, 2, 1, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00025316784740425646, 0.00026811144198291004, 0.0002702851197682321, 0.00025219927192665637, 0.0002497984387446195, 0.00022061709023546427, 0.00019892866839654744, 0.00018256594194099307, 0.00017686423962004483, 0.0001604434655746445, 0.00014481077960226685, 0.000133844354422763, 0.00013101348304189742, 0.00012030344805680215, 0.00011046518920920789, 0.00010146562999580055, 0.00010180410026805475, 9.191338904201984e-05, 8.35359314805828e-05, 7.67670790082775e-05, 8.004029223229736e-05, 7.34053974156268e-05, 6.654273602180183e-05, 6.0001370002282783e-05, 4.896382233710028e-05, 4.5895132643636316e-05, 4.3926629587076604e-05, 4.265817187842913e-05, 3.877907511196099e-05, 3.848329288302921e-05, 3.838892007479444e-05, 3.8790643884567544e-05 ] } } }, "pre_attention_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "afa8fa86c6ecedab7e0fd84b73ca7d5e942ca1b7238fcfa0893f3ce260e81e6b", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.04393457621335983, 0.024121688678860664, 0.030756350606679916, 0.03544071316719055, 0.03840360417962074, 0.026969486847519875, 0.027978016063570976, 0.029923241585493088, 0.03152087330818176, 0.024161184206604958, 0.025710709393024445, 0.02641460672020912, 0.02742084674537182, 0.021717891097068787, 0.0224718376994133, 0.023445574566721916, 0.024849263951182365, 0.01992032676935196, 0.02058842033147812, 0.020979218184947968, 0.02214338816702366, 0.01903437450528145, 0.02008410543203354, 0.021635910496115685, 0.023218298330903053, 0.022257626056671143, 0.024358727037906647, 0.025900650769472122, 0.027197737246751785, 0.02823900803923607, 0.02913021855056286, 0.029048049822449684 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 2.2108061784820165e-05, "first_to_last_ratio": 4.5038225950610284, "imbalance_abs_log_ratio": 1.5049265017537072, "last_quartile_mean": 4.908732819330908e-06, "mean": 1.424826272966584e-05, "normalized": [ 1.5393565072014856, 1.6436990565699428, 1.5746209905623882, 1.4393049982912232, 1.2296080351406573, 1.5542168455156227, 1.5210128581976592, 1.9112374884411145, 1.6656665742649008, 1.4395591772020961, 1.6324250537476421, 1.3835491754103626, 1.3477025444685555, 1.5824225341301699, 0.9384151980613074, 0.8636925984699546, 0.8034013225116797, 0.8830691126060858, 0.8605378947700126, 0.6634146810472719, 0.6587076476302224, 0.7136661453539468, 0.7077576665676224, 0.6868400571413836, 0.5793897205721035, 0.6202776091342075, 0.43119680892404477, 0.3692582531909216, 0.260043154103081, 0.12977534522113043, 0.15751431041740122, 0.20866063513380237 ], "peak_layer": 8, "peak_normalized": 1.9112374884411145, "population_cv": 0.5186010202163185, "rest_mean": 1.5120910120458575e-05, "spike_contrast": 0.6306476755313104, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.535966819385067e-06, "top_five_layers": [ 8, 9, 2, 11, 14 ], "top_five_spike_overlap": 0 }, "values": [ 2.1933155949227512e-05, 2.341985600651242e-05, 2.243561357317958e-05, 2.050759576377459e-05, 1.751977833919227e-05, 2.2144889953779057e-05, 2.167179081880022e-05, 2.723181387409568e-05, 2.3732854970148765e-05, 2.051121737167705e-05, 2.3259221052285284e-05, 1.9713172150659375e-05, 1.9202419935027137e-05, 2.2546772015630268e-05, 1.3370786291488912e-05, 1.2306119060667697e-05, 1.1447073120507412e-05, 1.2582200724864379e-05, 1.2261170013516676e-05, 9.452506674278993e-06, 9.385439625475556e-06, 1.0168502740270924e-05, 1.0084317182190716e-05, 9.786277587409131e-06, 8.255296961579006e-06, 8.837878340273164e-06, 6.143805421743309e-06, 5.26128860656172e-06, 3.705163180711679e-06, 1.8490732145437505e-06, 2.244305278509273e-06, 2.9730515507253585e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 6.226752566362848e-05, "first_to_last_ratio": 3.2558095109526493, "imbalance_abs_log_ratio": 1.1804409424216962, "last_quartile_mean": 1.9125051835544582e-05, "mean": 4.323053209986938e-05, "normalized": [ 1.0342606051842873, 1.898717126163711, 1.8089935381392723, 1.359356421079101, 1.274089744043777, 1.682360782557305, 1.2682690495969076, 1.1968320759706532, 1.2588398746694436, 1.2880627600055157, 1.2248689931301155, 1.273330768170305, 1.3149263319378108, 1.3464942246682916, 1.1005868831933079, 1.0691381511055633, 1.0591169598492518, 1.0073228103092708, 0.9327165506900205, 0.7338249833201036, 0.7729257473598179, 0.8419776567654331, 0.8893297062436448, 0.8244833428272487, 0.7679470877937514, 0.8196268734117844, 0.5402279437877997, 0.47239671566976066, 0.3242697700833044, 0.15928933562535502, 0.20350298326020644, 0.25191420338787907 ], "peak_layer": 2, "peak_normalized": 1.898717126163711, "population_cv": 0.42885792050879057, "rest_mean": 4.467689194387748e-05, "spike_contrast": 0.7928078118486851, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.542018894222565e-05, "top_five_layers": [ 2, 3, 6, 4, 14 ], "top_five_spike_overlap": 0 }, "values": [ 4.471163629204966e-05, 8.208255167119205e-05, 7.82037532189861e-05, 5.8765701396623626e-05, 5.507957757799886e-05, 7.272935181390494e-05, 5.482794585986994e-05, 5.173968747840263e-05, 5.442031761049293e-05, 5.568363849306479e-05, 5.295173832564615e-05, 5.50467666471377e-05, 5.684496500180103e-05, 5.820966180181131e-05, 4.757895658258349e-05, 4.621941116056405e-05, 4.578618973027915e-05, 4.354710108600557e-05, 4.0321832784684375e-05, 3.172364449710585e-05, 3.341399133205414e-05, 3.639914211817086e-05, 3.8446196413133293e-05, 3.564285361790098e-05, 3.319876122986898e-05, 3.543290586094372e-05, 2.3354341465164907e-05, 2.0421961380634457e-05, 1.4018354704603553e-05, 6.886162736918777e-06, 8.797542250249535e-06, 1.0890385055972729e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.05521946307271719, "first_to_last_ratio": 3.255809416547513, "imbalance_abs_log_ratio": 1.180440913425793, "last_quartile_mean": 0.016960287292022258, "mean": 0.03833726760058198, "normalized": [ 1.0342606436254083, 1.89871698336364, 1.808993472457683, 1.3593564915846248, 1.2740897521241208, 1.682360722725715, 1.2682690813216775, 1.1968320831111716, 1.2588398492110833, 1.2880628213558492, 1.224868979767805, 1.273330745435787, 1.314926371992558, 1.3464941899184253, 1.1005869061077207, 1.069138220456837, 1.059116922317307, 1.0073228539527392, 0.932716509673011, 0.7338250308305378, 0.7729257302043212, 0.841977653627328, 0.8893297206164247, 0.8244832831061564, 0.7679470991481019, 0.8196269423990945, 0.5402279418758984, 0.4723966869996653, 0.32426976863400875, 0.15928933802210024, 0.20350299592243773, 0.25191420811076204 ], "peak_layer": 2, "peak_normalized": 1.89871698336364, "population_cv": 0.42885790694027637, "rest_mean": 0.039619914031829, "spike_contrast": 0.7928077998002174, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.03141097687184811, "top_five_layers": [ 2, 3, 6, 4, 14 ], "top_five_spike_overlap": 0 }, "values": [ 0.039650727063417435, 0.07279162108898163, 0.06935186684131622, 0.05211401358246803, 0.04884511977434158, 0.06449711322784424, 0.048621971160173416, 0.04588327184319496, 0.04826048016548157, 0.04938080906867981, 0.04695812985301018, 0.0488160215318203, 0.0504106841981411, 0.051620908081531525, 0.042193494737148285, 0.04098783805966377, 0.040603648871183395, 0.03861800581216812, 0.03575780242681503, 0.02813284657895565, 0.029631860554218292, 0.032279122620821, 0.034094471484422684, 0.03160843625664711, 0.029440993443131447, 0.03142225742340088, 0.020710863173007965, 0.018110398203134537, 0.012431616894900799, 0.006106717977672815, 0.007801748812198639, 0.009657702408730984 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 1.2073289212821692e-05, "first_to_last_ratio": 5.118173394564018, "imbalance_abs_log_ratio": 1.6327976165236144, "last_quartile_mean": 2.358905859978222e-06, "mean": 9.608797252980139e-06, "normalized": [ 2.2116042312522914, 1.2252124207577697, 0.9166421564982928, 1.186820314517719, 0.9292558997947784, 1.1346790160633606, 1.0583515009014062, 1.3892974727904446, 1.5862793081829674, 1.2655275776533215, 1.4193695477429429, 2.042377666614073, 1.9979815760663124, 2.116845169548275, 1.4502152158936208, 1.0590931016242418, 1.4220084536973343, 1.379051361973986, 1.173597294405293, 0.9241153337121745, 0.6121286422299292, 0.5710108338819657, 0.461893587104583, 0.5026871640197357, 0.30513715456990576, 0.2951723976971137, 0.2866930192727529, 0.2870900856457325, 0.21497083513437823, 0.17484004907583897, 0.18455905309901366, 0.21549255857844585 ], "peak_layer": 1, "peak_normalized": 2.2116042312522914, "population_cv": 0.6038429581123452, "rest_mean": 1.0515277882337546e-05, "spike_contrast": 0.44828124441370076, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.713801854450139e-06, "top_five_layers": [ 1, 14, 12, 13, 9 ], "top_five_spike_overlap": 0 }, "values": [ 2.1250856661936268e-05, 1.1772817742894404e-05, 8.807828635326587e-06, 1.1403915777918883e-05, 8.929031537263654e-06, 1.0902900612563826e-05, 1.0169484994548839e-05, 1.3349477740121074e-05, 1.5242236258927733e-05, 1.2160197911725845e-05, 1.3638434211316053e-05, 1.9624792912509292e-05, 1.919819987961091e-05, 2.034033605013974e-05, 1.3934823982708622e-05, 1.017661088553723e-05, 1.3663790923601482e-05, 1.3251024938654155e-05, 1.1276858458586503e-05, 8.879636880010366e-06, 5.8818200159294065e-06, 5.4867273320269305e-06, 4.43824183093966e-06, 4.830219040741213e-06, 2.9320010526134865e-06, 2.836251724147587e-06, 2.75477509603661e-06, 2.7585904263105476e-06, 2.06561117011006e-06, 1.6800025832708343e-06, 1.773390522430418e-06, 2.0706243049062323e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 4.736539585792343e-05, "first_to_last_ratio": 5.693189725390274, "imbalance_abs_log_ratio": 1.7392706754966003, "last_quartile_mean": 8.319658775235439e-06, "mean": 3.0428698622131378e-05, "normalized": [ 0.9332234373969145, 2.093244562439342, 2.000219271237334, 1.4660610383042352, 1.4267640458493276, 1.8877254776078727, 1.397617962458032, 1.2479661794130616, 1.4262249611304942, 1.4477791439571595, 1.24181841376056, 1.2462360629867504, 1.2738256274260629, 1.4269714780931013, 1.1746071387256543, 1.0297217541028905, 1.0028420024769986, 1.0837416535661664, 0.9989109522318798, 0.7645609375506638, 0.8126373951104926, 0.8905222845518224, 0.8066853450678224, 0.7327738890015353, 0.50846646429871, 0.4349609059082798, 0.29076653368375877, 0.2846228030965179, 0.21016872042077955, 0.130190170117833, 0.14875731368946996, 0.17938607433847711 ], "peak_layer": 2, "peak_normalized": 2.093244562439342, "population_cv": 0.5294278422744164, "rest_mean": 3.183621146052718e-05, "spike_contrast": 0.7170491791430023, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.2828129294794054e-05, "top_five_layers": [ 2, 3, 6, 4, 10 ], "top_five_spike_overlap": 0 }, "values": [ 2.83967747236602e-05, 6.369470793288201e-05, 6.086406938266009e-05, 4.461032949620858e-05, 4.3414573156042024e-05, 5.744102963944897e-05, 4.2527695768512785e-05, 3.797398676397279e-05, 4.339816950960085e-05, 4.405403524287976e-05, 3.7786918255733326e-05, 3.7921341572655365e-05, 3.876085611409508e-05, 4.342088504927233e-05, 3.5741766623687e-05, 3.133309292024933e-05, 3.051517705898732e-05, 3.297684816061519e-05, 3.0395560315810144e-05, 2.3264594346983358e-05, 2.4727498384891078e-05, 2.7097434212919325e-05, 2.4546385247958824e-05, 2.2297355826594867e-05, 1.547197280160617e-05, 1.323529431829229e-05, 8.847647222864907e-06, 8.660701496410184e-06, 6.39516065348289e-06, 3.961517450079555e-06, 4.52649146609474e-06, 5.458484793052776e-06 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 3.615552941482747e-05, "first_to_last_ratio": 8.764492263828256, "imbalance_abs_log_ratio": 2.1707085890088735, "last_quartile_mean": 4.1252280595927004e-06, "mean": 2.208462114339227e-05, "normalized": [ 0.7494933219758998, 2.2634292925463795, 2.1410570182439597, 1.5739498957546751, 1.5487374547204664, 2.0376524711677733, 1.534141636657325, 1.2486273054515569, 1.5321016320199576, 1.6393386005987565, 1.2261935965091952, 1.1547399796224558, 1.219246066181087, 1.4546728804264264, 1.2279261343623535, 1.0312465593605784, 0.9063241145323925, 1.1172138768757267, 1.0304062764426833, 0.7847973944776799, 0.8360866099062609, 0.911401228786939, 0.6978180640030948, 0.6390634924568735, 0.31508360869438684, 0.26042370730894115, 0.18982145136915554, 0.23533143612154772, 0.14063662904795424, 0.10306654364310344, 0.11411045319594963, 0.13586126753846497 ], "peak_layer": 2, "peak_normalized": 2.2634292925463795, "population_cv": 0.6059056054769965, "rest_mean": 2.3393786848289352e-05, "spike_contrast": 0.6418424872519509, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.5015126336948015e-05, "top_five_layers": [ 2, 3, 6, 10, 4 ], "top_five_spike_overlap": 0 }, "values": [ 1.6552276065340266e-05, 4.998697841074318e-05, 4.7284433094318956e-05, 3.476008714642376e-05, 3.420327993808314e-05, 4.500078284763731e-05, 3.388093682588078e-05, 2.757546099019237e-05, 3.383588409633376e-05, 3.620417191996239e-05, 2.708002102735918e-05, 2.5501994969090447e-05, 2.6926587452180684e-05, 3.212589945178479e-05, 2.7118283469462767e-05, 2.277468956890516e-05, 2.0015824702568352e-05, 2.467324520694092e-05, 2.2756132239010185e-05, 1.7331953131360933e-05, 1.8464656022842973e-05, 2.0127950847381726e-05, 1.5411047570523806e-05, 1.4113475117483176e-05, 6.958502126508392e-06, 5.7513589126756415e-06, 4.19213483837666e-06, 5.1972056098748e-06, 3.1059066714078654e-06, 2.276185568916844e-06, 2.520086127333343e-06, 3.0004446216480574e-06 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00011569616435735952, "first_to_last_ratio": 4.483785350026755, "imbalance_abs_log_ratio": 1.500467633864892, "last_quartile_mean": 2.5803234393606544e-05, "mean": 7.895687571135568e-05, "normalized": [ 1.113643291734891, 1.9315033327137208, 1.8689084538217502, 1.3013119102609747, 1.3204401608174607, 1.7172337521094045, 1.2105960380108525, 1.2588293628008012, 1.3316624978487503, 1.2436188212370092, 1.3620564885728002, 1.3794695319626817, 1.4052872102652663, 1.3953280795865155, 1.1319795963186805, 0.9796315470123751, 1.1579414909870167, 1.0808582581724016, 0.9982944348550936, 0.7435043404975695, 0.8011357344053238, 0.8526183967497576, 0.9599446768973036, 0.8397896829211625, 0.7011434784757186, 0.5094988193534132, 0.32142473628464113, 0.2742399520171878, 0.26163364470536055, 0.152278590465612, 0.18046309892286552, 0.21373058921563795 ], "peak_layer": 2, "peak_normalized": 1.9315033327137208, "population_cv": 0.47743629272692334, "rest_mean": 8.142900973056115e-05, "spike_contrast": 0.8056999860950416, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.560735200764612e-05, "top_five_layers": [ 2, 3, 6, 13, 14 ], "top_five_spike_overlap": 0 }, "values": [ 8.79297949722968e-05, 0.00015250546857714653, 0.00014756317250430584, 0.00010274752276018262, 0.00010425782966194674, 0.00013558741193264723, 9.558488090988249e-05, 9.939323354046792e-05, 0.00010514391033211723, 9.819225670071319e-05, 0.00010754372488008812, 0.00010891860438277945, 0.00011095708759967238, 0.00011017074575647712, 8.937757229432464e-05, 7.734864630037919e-05, 9.142744238488376e-05, 8.53411911521107e-05, 7.882220961619169e-05, 5.870477980352007e-05, 6.32551746093668e-05, 6.732008478138596e-05, 7.579423254355788e-05, 6.630716961808503e-05, 5.5360098485834897e-05, 4.022843495476991e-05, 2.5378692953381687e-05, 2.165312980650924e-05, 2.0657775166910142e-05, 1.2023441740893759e-05, 1.4248802472138777e-05, 1.6875499568413943e-05 ] } } }, "pre_mlp_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "65621f80daccd38bb1dc68831a9ef8c381d5d3b87c541e2988c6ff3556a7910d", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.018067941069602966, 0.027584951370954514, 0.03231721371412277, 0.036101218312978745, 0.025345133617520332, 0.027166159823536873, 0.028526438400149345, 0.02979987859725952, 0.02248864620923996, 0.023894935846328735, 0.02451697364449501, 0.025183402001857758, 0.020499059930443764, 0.02105838619172573, 0.0216353852301836, 0.022715488448739052, 0.01881815865635872, 0.0193756353110075, 0.02001863345503807, 0.021088071167469025, 0.018417155370116234, 0.019219009205698967, 0.020342160016298294, 0.02150510996580124, 0.02099662832915783, 0.02211105264723301, 0.02328667603433132, 0.024160878732800484, 0.02260657213628292, 0.024989914149045944, 0.02536177635192871, 0.027097374200820923 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.2429669027369528e-05, "first_to_last_ratio": 1.5167074865147177, "imbalance_abs_log_ratio": 0.4165418581180973, "last_quartile_mean": 8.195165605684451e-06, "mean": 1.1438181132916725e-05, "normalized": [ 2.3239055235296235, 1.1190114244144547, 0.8331917749372405, 0.7813452753113945, 1.003803456769089, 0.9365675136194498, 0.8842494964053754, 0.8113839672107822, 1.0847409217961617, 1.0682117256400467, 0.9883998597662562, 0.9038042770049443, 1.1938503302540149, 1.1817190494107024, 1.0722151723600664, 0.9843285081524176, 1.2863323428556752, 1.156982982287669, 1.0862279117485159, 1.0234351273362494, 1.2841852282798012, 1.1754321217296801, 1.070502203826642, 1.0143774573484126, 0.8953924191337335, 0.8270637955966683, 0.8133202903656793, 0.7381861493144678, 0.6730532700250402, 0.5804717059865397, 0.6082410306470007, 0.5960676869362054 ], "peak_layer": 1, "peak_normalized": 2.3239055235296235, "population_cv": 0.3045451785807339, "rest_mean": 1.1251827985419098e-05, "spike_contrast": 1.1059970118215763, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.2444488129403908e-05, "top_five_layers": [ 1, 17, 21, 13, 14 ], "top_five_spike_overlap": 1 }, "values": [ 2.6581252313917503e-05, 1.2799455362255685e-05, 9.530198440188542e-06, 8.937168786360417e-06, 1.1481685760372784e-05, 1.0712628863984719e-05, 1.011420590657508e-05, 9.280756785301492e-06, 1.2407463145791553e-05, 1.22183992061764e-05, 1.1305496627755929e-05, 1.0337877029087394e-05, 1.3655476323037874e-05, 1.3516716535377782e-05, 1.2264191354915965e-05, 1.125892777054105e-05, 1.4713302334712353e-05, 1.323378091910854e-05, 1.2424471606209408e-05, 1.1706236364261713e-05, 1.4688743249280378e-05, 1.3444805517792702e-05, 1.2244598110555671e-05, 1.1602633094298653e-05, 1.0241660675092135e-05, 9.460105502512306e-06, 9.302904800279066e-06, 8.443506885669194e-06, 7.698505214648321e-06, 6.639540515607223e-06, 6.957171081012348e-06, 6.817930170655018e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 4.8672892262402456e-05, "first_to_last_ratio": 1.4790455122350374, "imbalance_abs_log_ratio": 0.39139695555746173, "last_quartile_mean": 3.290831273261574e-05, "mean": 4.5425740495375067e-05, "normalized": [ 2.292867781479936, 1.098844858197334, 0.8288263395703189, 0.7584336983627002, 0.9778795698933019, 0.9351432056276012, 0.8751120479081964, 0.8047536035443427, 1.088349470440326, 1.0783459726467945, 0.9909402014503932, 0.9019584923345375, 1.2027204540239667, 1.175772139842897, 1.0782465054806212, 0.9953149147774686, 1.2799515830354413, 1.1648392407100392, 1.0907985090406367, 1.0337452797487645, 1.2844092657539465, 1.1758215530808431, 1.0768851187191208, 1.0145044696518641, 0.9072210422189393, 0.8359231056438284, 0.8227296910259355, 0.7472276249111484, 0.6761454407882689, 0.5893866866897062, 0.6130538257244701, 0.6038483076763113 ], "peak_layer": 1, "peak_normalized": 2.292867781479936, "population_cv": 0.29963351102833546, "rest_mean": 4.465376965796437e-05, "spike_contrast": 1.1106426937137044, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.9594383017392826e-05, "top_five_layers": [ 1, 21, 17, 13, 22 ], "top_five_spike_overlap": 2 }, "values": [ 0.00010415521683171391, 4.9915841373149306e-05, 3.765005021705292e-05, 3.445241236477159e-05, 4.442090357770212e-05, 4.2479572584852576e-05, 3.975261279265396e-05, 3.655652835732326e-05, 4.943908061250113e-05, 4.898466431768611e-05, 4.501419243752025e-05, 4.097213241038844e-05, 5.4634467232972383e-05, 5.3410320106195286e-05, 4.89801459480077e-05, 4.5212917029857635e-05, 5.814274845761247e-05, 5.291368506732397e-05, 4.9550330004421994e-05, 4.695864481618628e-05, 5.8345241995994e-05, 5.341256473911926e-05, 4.891830394626595e-05, 4.608461676980369e-05, 4.1211187635781243e-05, 3.797242607106455e-05, 3.737310544238426e-05, 3.394336818018928e-05, 3.071440733037889e-05, 2.6773326680995524e-05, 2.784842399705667e-05, 2.7430256523075514e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.0431636068969965, "first_to_last_ratio": 1.4790455355871048, "imbalance_abs_log_ratio": 0.39139697134606793, "last_quartile_mean": 0.02918341988697648, "mean": 0.04028400045353919, "normalized": [ 2.2928676956325664, 1.0988448461233453, 0.8288263129280055, 0.7584337335279033, 0.9778795372444193, 0.9351432892874852, 0.8751120548755073, 0.8047535990009899, 1.0883495038762179, 1.0783460397661777, 0.9909401548295442, 0.9019584843211922, 1.2027204645548106, 1.1757721269107668, 1.0782465359354751, 0.9953149017803241, 1.2799516204237265, 1.1648392822258575, 1.090798537285597, 1.0337452971674015, 1.2844092255829724, 1.1758215089234016, 1.0768851089878908, 1.014504529948594, 0.9072209977912502, 0.8359230814928427, 0.8227296688325424, 0.7472276232112787, 0.676145406434352, 0.5893866663000006, 0.6130538743742001, 0.6038482904233612 ], "peak_layer": 1, "peak_normalized": 2.2928676956325664, "population_cv": 0.29963350187956755, "rest_mean": 0.0395994091199504, "spike_contrast": 1.110642674533265, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.04398079365491867, "top_five_layers": [ 1, 21, 17, 13, 22 ], "top_five_spike_overlap": 2 }, "values": [ 0.09236588329076767, 0.04426586627960205, 0.03338843956589699, 0.03055274486541748, 0.039392899721860886, 0.03767131268978119, 0.03525301441550255, 0.03241869434714317, 0.04384307190775871, 0.043440092355012894, 0.03991903364658356, 0.03633449599146843, 0.04845039173960686, 0.04736480489373207, 0.04343608394265175, 0.040095265954732895, 0.05156157165765762, 0.04692438617348671, 0.04394172877073288, 0.04164339601993561, 0.05174114182591438, 0.04736679419875145, 0.04338124021887779, 0.04086830094456673, 0.036546491086483, 0.03367432579398155, 0.03314284235239029, 0.030101317912340164, 0.027237841859459877, 0.023742852732539177, 0.024696262553334236, 0.024325424805283546 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 3.470277960104795e-06, "first_to_last_ratio": 1.5769587704873496, "imbalance_abs_log_ratio": 0.4554981633698566, "last_quartile_mean": 2.2006142614827695e-06, "mean": 2.887214563429552e-06, "normalized": [ 2.7519535335351715, 1.313168413337379, 0.9399430071407856, 0.8753790372729023, 1.0813834937551676, 0.9567319703631147, 0.8229162441441861, 0.8740977436183472, 1.0566599601043785, 0.9454608388064315, 0.9224389765316489, 0.8092920053718847, 1.0583487949182648, 1.0534209725986503, 0.9874597862575576, 0.8539122103086818, 1.169801497382018, 1.081662118029895, 1.0127159978603695, 0.961060648114576, 1.1999729309980693, 1.1228483457198037, 1.0226342502719021, 1.0291945210343731, 0.8967756962809945, 0.8510820239929227, 0.8261498765197046, 0.7721004685625728, 0.7451436093586472, 0.6285081646269207, 0.6879375096468019, 0.6898453535358771 ], "peak_layer": 1, "peak_normalized": 2.7519535335351715, "population_cv": 0.35016697743385605, "rest_mean": 2.8581899574352e-06, "spike_contrast": 1.0649912990844543, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.043947435799055e-06, "top_five_layers": [ 1, 2, 21, 17, 22 ], "top_five_spike_overlap": 2 }, "values": [ 7.945480319904163e-06, 3.7913989672233583e-06, 2.7138171390106436e-06, 2.5274071049352642e-06, 3.12218617182225e-06, 2.7622904781310353e-06, 2.3759357645758428e-06, 2.523707735235803e-06, 3.0508040254062507e-06, 2.729748302954249e-06, 2.6632792469172273e-06, 2.3365996639768127e-06, 3.055680053876131e-06, 3.041452373508946e-06, 2.851008275683853e-06, 2.4654277694935445e-06, 3.3774679195630597e-06, 3.122990619885968e-06, 2.92392837764055e-06, 2.774788299575448e-06, 3.4645793220988708e-06, 3.2419040962849976e-06, 2.952564500446897e-06, 2.9715054097323446e-06, 2.589183850432164e-06, 2.4572564143454656e-06, 2.3852719550632173e-06, 2.229219717264641e-06, 2.1513894807867473e-06, 1.814637926145224e-06, 1.9862231965817045e-06, 1.9917315512429923e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 4.1570810026314575e-05, "first_to_last_ratio": 1.5044497617163743, "imbalance_abs_log_ratio": 0.40842722451739255, "last_quartile_mean": 2.763190309451602e-05, "mean": 3.7813772848949156e-05, "normalized": [ 2.388009071075982, 1.1470596649494917, 0.8534302761319221, 0.7776431816025992, 0.9983567968228081, 0.9421045940312636, 0.8787540856715802, 0.8094928604716568, 1.0777400416324399, 1.0654722038935143, 0.9718539610468895, 0.8865810688989143, 1.168850821337844, 1.1332939684204937, 1.0417735360252487, 0.9621498679953465, 1.2282401671332226, 1.126146404473492, 1.059853477638917, 1.0196054375892694, 1.2877065753533457, 1.1949291884271547, 1.0946140188339069, 1.0404469270679417, 0.9334039480335107, 0.8520038997158634, 0.8392353967355263, 0.7615998622978171, 0.6682831346061853, 0.5860512257947129, 0.6091233405529298, 0.5961909957382107 ], "peak_layer": 1, "peak_normalized": 2.388009071075982, "population_cv": 0.309055923286973, "rest_mean": 3.704195080866123e-05, "spike_contrast": 1.13335315635395, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.198161186650395e-05, "top_five_layers": [ 1, 21, 17, 22, 13 ], "top_five_spike_overlap": 2 }, "values": [ 9.029963257489726e-05, 4.337465361459181e-05, 3.227141860406846e-05, 2.9405622626654804e-05, 3.775163713726215e-05, 3.562452911864966e-05, 3.322900738567114e-05, 3.060997914872132e-05, 4.0753417124506086e-05, 4.0289523894898593e-05, 3.674946492537856e-05, 3.3524975151522085e-05, 4.419865945237689e-05, 4.2854120692936704e-05, 3.9393387851305306e-05, 3.638251655502245e-05, 4.644439468393102e-05, 4.258384433342144e-05, 4.007705865660682e-05, 3.855512841255404e-05, 4.869304393650964e-05, 4.518478090176359e-05, 4.139148586546071e-05, 3.934322376153432e-05, 3.5295524867251515e-05, 3.2217481930274516e-05, 3.173465665895492e-05, 2.879896419472061e-05, 2.5270306650782004e-05, 2.2160807930049486e-05, 2.303325163666159e-05, 2.2544230887433514e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 3.563517248039716e-05, "first_to_last_ratio": 1.4352355648633195, "imbalance_abs_log_ratio": 0.3613289924385252, "last_quartile_mean": 2.4828796995279845e-05, "mean": 3.227010705586508e-05, "normalized": [ 2.4014447290909864, 1.1673853090110469, 0.8593097269193264, 0.7832036205871891, 1.0067937101766118, 0.9335226700834045, 0.8775072266073977, 0.8050578010660164, 1.053006259394381, 1.0264307263592312, 0.9338631869266799, 0.8595966381479759, 1.1212921464302679, 1.0758485643993823, 0.9942738962286088, 0.9189156755903263, 1.1746749961886236, 1.0800838025282244, 1.019430881526982, 0.9950392558872734, 1.2898668670070823, 1.2311059818213306, 1.13592608464973, 1.1011764563471198, 0.9942343261534591, 0.8955937948027416, 0.9014365249588188, 0.831025803802073, 0.6870790019632969, 0.6046383015495036, 0.6312332814164576, 0.610002752378451 ], "peak_layer": 1, "peak_normalized": 2.4014447290909864, "population_cv": 0.30444570497962015, "rest_mean": 3.1370954645724315e-05, "spike_contrast": 1.1834364140297289, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.7125530070625246e-05, "top_five_layers": [ 1, 21, 22, 17, 2 ], "top_five_spike_overlap": 2 }, "values": [ 7.749487849650905e-05, 3.7671648897230625e-05, 2.7730016881832853e-05, 2.527406468288973e-05, 3.2489340810570866e-05, 3.0124876502668485e-05, 2.8317252144915983e-05, 2.5979301426559687e-05, 3.398062472115271e-05, 3.312302942504175e-05, 3.0135865017655306e-05, 2.77392755378969e-05, 3.618421760620549e-05, 3.4717748349066824e-05, 3.2085325074149296e-05, 2.965350722661242e-05, 3.790688788285479e-05, 3.4854419936891645e-05, 3.289714368293062e-05, 3.211002331227064e-05, 4.162414188613184e-05, 3.972792183049023e-05, 3.6656456359196454e-05, 3.55350821337197e-05, 3.2084048143588006e-05, 2.8900907636852935e-05, 2.9089453164488077e-05, 2.6817291654879227e-05, 2.2172112949192524e-05, 1.9511742721078917e-05, 2.03699655685341e-05, 1.9684854123624973e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 8.83701450220542e-05, "first_to_last_ratio": 1.5137756203780361, "imbalance_abs_log_ratio": 0.41460694084716315, "last_quartile_mean": 5.837730759594706e-05, "mean": 8.360470872048609e-05, "normalized": [ 2.260874919572962, 1.046876661671232, 0.8108813008392315, 0.7410270778432692, 0.9650502462422242, 0.9363082641888973, 0.8839048255705971, 0.8110736329048943, 1.115418327104151, 1.1154856868411749, 1.0274711394986145, 0.9306550068796721, 1.2562444747790198, 1.2247146350825069, 1.1283567053258787, 1.0398138975133757, 1.305825331548614, 1.176867029634062, 1.1082958616806637, 1.034638772497703, 1.2882363465735436, 1.1524683581694375, 1.0651417678309785, 0.9883391355795494, 0.8670300800809992, 0.8013878422949675, 0.7767996236633684, 0.7006926068874338, 0.6674164616571702, 0.5747719332171776, 0.5954528519609661, 0.6024791948656639 ], "peak_layer": 1, "peak_normalized": 2.260874919572962, "population_cv": 0.30484696934594563, "rest_mean": 8.24862149278892e-05, "spike_contrast": 1.0867825039478176, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 8.964457520050928e-05, "top_five_layers": [ 1, 17, 21, 13, 14 ], "top_five_spike_overlap": 1 }, "values": [ 0.0001890197891043499, 8.752381836529821e-05, 6.77934949635528e-05, 6.195335299707949e-05, 8.068274473771453e-05, 7.82797797000967e-05, 7.389860547846183e-05, 6.780957482988015e-05, 9.325422433903441e-05, 9.325985593022779e-05, 8.590142533648759e-05, 7.780714076943696e-05, 0.00010502795339561999, 0.0001023919103317894, 9.433593368157744e-05, 8.693333802511916e-05, 0.00010917314648395404, 9.839162521529943e-05, 9.265875269193202e-05, 8.650067320559174e-05, 0.00010770262451842427, 9.635178139433265e-05, 8.905086724553257e-05, 8.262980554718524e-05, 7.248779729707167e-05, 6.69997971272096e-05, 6.494410627055913e-05, 5.858120130142197e-05, 5.579915887210518e-05, 4.8053640057332814e-05, 4.978266224497929e-05, 5.037009759689681e-05 ] } } } } }, "parent_learned_round06_exact": true, "step": 500 }, { "local_matrix": null, "parent_learned": { "bits_per_byte": 2.073383110809672, "forward": { "activation_sha256": { "attention_branch_output": "e88f25858f6afc8c2818e9d252ea08bdd359890e32905783ed1bf3fce3e953f1", "mlp_branch_output": "dba9c602cebba42dc99cd260f60389d5962764f3da1c351817656a50dba8a4c1", "post_attention_state": "461f23fe7003cf2bffe4fbf4022ddcdfd511968996b161f3081b3d083dec0aab", "post_mlp_state": "6637e7e2c52ffc1501509e8cf074d32495c0b5360809f4fcf8995d260c2092c2", "pre_attention_input": "9f622711336f6114d5391a2fcd50030068888edcdff338c4d870b49ef8b7290b", "pre_mlp_input": "150a36c8e5b01ef896c4498cdc7a2002d1b68d6cfc87a698c943b18cb0623a93" }, "logits_sha256": "9adeddfaa352b7946853a2fe6e55c7269e6a01f6272498cdfb911b26410503c6", "loss_nats": 1.4371596574783325, "mixer_summary_sha256": "739af9a4ced9bd8d3e1cd2555b418bfc5db56ce4da6de7d1c45506ae68b1e3e1" }, "loss_nats": 1.4371596574783325, "loss_scale": 1.0, "mixers": [ { "branch": "attention", "entropy_mean": -0.0, "group": 1, "latest_source_mass": 1.0, "layer": 1, "max_source_mass": 1.0, "mixer_index": 0, "normalized_entropy": 1.0, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 1.0, "median": 1.0, "p05": 1.0, "p95": 1.0 } ], "sources": 1 }, { "branch": "mlp", "entropy_mean": 0.4738388657569885, "group": 1, "latest_source_mass": 0.19323334097862244, "layer": 1, "max_source_mass": 0.8067666888237, "mixer_index": 1, "normalized_entropy": 0.6836049818080586, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.8067666888237, "median": 0.8265105485916138, "p05": 0.6604017019271851, "p95": 0.8949732184410095 }, { "label": "current_group_partial", "mean": 0.19323334097862244, "median": 0.17348942160606384, "p05": 0.10502681136131287, "p95": 0.33959829807281494 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6344536542892456, "group": 1, "latest_source_mass": 0.5104025602340698, "layer": 2, "max_source_mass": 0.5104025602340698, "mixer_index": 2, "normalized_entropy": 0.9153231407169755, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.4895974397659302, "median": 0.45956116914749146, "p05": 0.2515213191509247, "p95": 0.7871550917625427 }, { "label": "current_group_partial", "mean": 0.5104025602340698, "median": 0.5404388308525085, "p05": 0.21284496784210205, "p95": 0.7484787106513977 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.41773349046707153, "group": 1, "latest_source_mass": 0.15382957458496094, "layer": 2, "max_source_mass": 0.8461704254150391, "mixer_index": 3, "normalized_entropy": 0.6026620351100812, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.8461704254150391, "median": 0.8643104434013367, "p05": 0.7285550832748413, "p95": 0.9108630418777466 }, { "label": "current_group_partial", "mean": 0.15382957458496094, "median": 0.13568954169750214, "p05": 0.08913692831993103, "p95": 0.2714449167251587 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6868985891342163, "group": 1, "latest_source_mass": 0.49918031692504883, "layer": 3, "max_source_mass": 0.500819742679596, "mixer_index": 4, "normalized_entropy": 0.9909851881375595, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.500819742679596, "median": 0.50103759765625, "p05": 0.41182032227516174, "p95": 0.5924672484397888 }, { "label": "current_group_partial", "mean": 0.49918031692504883, "median": 0.49896240234375, "p05": 0.40753278136253357, "p95": 0.5881797075271606 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.5841774344444275, "group": 1, "latest_source_mass": 0.315071702003479, "layer": 3, "max_source_mass": 0.684928297996521, "mixer_index": 5, "normalized_entropy": 0.8427898876722131, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.684928297996521, "median": 0.6976089477539062, "p05": 0.4487886428833008, "p95": 0.8860265612602234 }, { "label": "current_group_partial", "mean": 0.315071702003479, "median": 0.3023911118507385, "p05": 0.1139734610915184, "p95": 0.5512113571166992 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6896515488624573, "group": 1, "latest_source_mass": 0.49074798822402954, "layer": 4, "max_source_mass": 0.5092520117759705, "mixer_index": 6, "normalized_entropy": 0.9949568694852597, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.5092520117759705, "median": 0.509276270866394, "p05": 0.4409506916999817, "p95": 0.5760948657989502 }, { "label": "current_group_partial", "mean": 0.49074798822402954, "median": 0.49072372913360596, "p05": 0.4239051342010498, "p95": 0.5590493679046631 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.5611414909362793, "group": 1, "latest_source_mass": 0.2719496786594391, "layer": 4, "max_source_mass": 0.7280503511428833, "mixer_index": 7, "normalized_entropy": 0.8095560462108093, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.7280503511428833, "median": 0.7264741063117981, "p05": 0.5577248930931091, "p95": 0.8873021602630615 }, { "label": "current_group_partial", "mean": 0.2719496786594391, "median": 0.2735258936882019, "p05": 0.11269781738519669, "p95": 0.44227516651153564 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6817618608474731, "group": 2, "latest_source_mass": 0.4686949551105499, "layer": 5, "max_source_mass": 0.5313050746917725, "mixer_index": 8, "normalized_entropy": 0.9835744557118811, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.5313050746917725, "median": 0.532880961894989, "p05": 0.4224134683609009, "p95": 0.6363531351089478 }, { "label": "completed_group_1", "mean": 0.4686949551105499, "median": 0.4671190679073334, "p05": 0.363646924495697, "p95": 0.5775865316390991 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 1.0218309164047241, "group": 2, "latest_source_mass": 0.2602040767669678, "layer": 5, "max_source_mass": 0.4974328279495239, "mixer_index": 9, "normalized_entropy": 0.9301105830916285, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.4974328279495239, "median": 0.4930803179740906, "p05": 0.35206541419029236, "p95": 0.6587448716163635 }, { "label": "completed_group_1", "mean": 0.2423631250858307, "median": 0.23208677768707275, "p05": 0.15309149026870728, "p95": 0.3585616946220398 }, { "label": "current_group_partial", "mean": 0.2602040767669678, "median": 0.261092871427536, "p05": 0.1870650351047516, "p95": 0.33141374588012695 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0932960510253906, "group": 2, "latest_source_mass": 0.34662574529647827, "layer": 6, "max_source_mass": 0.3533782958984375, "mixer_index": 10, "normalized_entropy": 0.9951609519595268, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.3533782958984375, "median": 0.3537653684616089, "p05": 0.300750195980072, "p95": 0.40760302543640137 }, { "label": "completed_group_1", "mean": 0.29999595880508423, "median": 0.2998189330101013, "p05": 0.277343213558197, "p95": 0.3256012797355652 }, { "label": "current_group_partial", "mean": 0.34662574529647827, "median": 0.34948140382766724, "p05": 0.3000357747077942, "p95": 0.3795154392719269 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0456881523132324, "group": 2, "latest_source_mass": 0.3350909352302551, "layer": 6, "max_source_mass": 0.4338536858558655, "mixer_index": 11, "normalized_entropy": 0.9518263750544431, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.4338536858558655, "median": 0.43320339918136597, "p05": 0.2745198607444763, "p95": 0.6123260259628296 }, { "label": "completed_group_1", "mean": 0.23105540871620178, "median": 0.22906750440597534, "p05": 0.15816110372543335, "p95": 0.3054558038711548 }, { "label": "current_group_partial", "mean": 0.3350909352302551, "median": 0.33941850066185, "p05": 0.2199106216430664, "p95": 0.4273326098918915 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0891339778900146, "group": 2, "latest_source_mass": 0.37785181403160095, "layer": 7, "max_source_mass": 0.37785181403160095, "mixer_index": 12, "normalized_entropy": 0.9913724697276178, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.33998239040374756, "median": 0.3426361680030823, "p05": 0.2884981632232666, "p95": 0.38276687264442444 }, { "label": "completed_group_1", "mean": 0.2821658253669739, "median": 0.2775929868221283, "p05": 0.2536381781101227, "p95": 0.3205426335334778 }, { "label": "current_group_partial", "mean": 0.37785181403160095, "median": 0.3805788457393646, "p05": 0.34096643328666687, "p95": 0.40283167362213135 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.038641333580017, "group": 2, "latest_source_mass": 0.4193001687526703, "layer": 7, "max_source_mass": 0.4193001687526703, "mixer_index": 13, "normalized_entropy": 0.9454120842205417, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.34165239334106445, "median": 0.308063805103302, "p05": 0.2054935097694397, "p95": 0.6325749754905701 }, { "label": "completed_group_1", "mean": 0.23904743790626526, "median": 0.244267076253891, "p05": 0.1361692249774933, "p95": 0.30873122811317444 }, { "label": "current_group_partial", "mean": 0.4193001687526703, "median": 0.44425007700920105, "p05": 0.22311054170131683, "p95": 0.5117462277412415 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0754427909851074, "group": 2, "latest_source_mass": 0.3816372752189636, "layer": 8, "max_source_mass": 0.3816372752189636, "mixer_index": 14, "normalized_entropy": 0.9789102143476917, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.3781457245349884, "median": 0.37959107756614685, "p05": 0.3339657485485077, "p95": 0.42053842544555664 }, { "label": "completed_group_1", "mean": 0.24021697044372559, "median": 0.23580573499202728, "p05": 0.21105526387691498, "p95": 0.2833511531352997 }, { "label": "current_group_partial", "mean": 0.3816372752189636, "median": 0.38216716051101685, "p05": 0.3468884229660034, "p95": 0.4153265357017517 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0433809757232666, "group": 2, "latest_source_mass": 0.4086577892303467, "layer": 8, "max_source_mass": 0.4086577892303467, "mixer_index": 15, "normalized_entropy": 0.9497262924195011, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.37802231311798096, "median": 0.3628900945186615, "p05": 0.27504366636276245, "p95": 0.5507957339286804 }, { "label": "completed_group_1", "mean": 0.21331989765167236, "median": 0.2002047300338745, "p05": 0.13782089948654175, "p95": 0.3198853135108948 }, { "label": "current_group_partial", "mean": 0.4086577892303467, "median": 0.41623425483703613, "p05": 0.29855865240097046, "p95": 0.47824543714523315 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0922105312347412, "group": 3, "latest_source_mass": 0.3464643955230713, "layer": 9, "max_source_mass": 0.3464643955230713, "mixer_index": 16, "normalized_entropy": 0.994172869264798, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.3460429310798645, "median": 0.3446507453918457, "p05": 0.28670766949653625, "p95": 0.40976202487945557 }, { "label": "completed_group_1", "mean": 0.3074926435947418, "median": 0.306111603975296, "p05": 0.2859768867492676, "p95": 0.3337686061859131 }, { "label": "completed_group_2", "mean": 0.3464643955230713, "median": 0.3486866354942322, "p05": 0.27475792169570923, "p95": 0.40797942876815796 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.3404325246810913, "group": 3, "latest_source_mass": 0.3108460605144501, "layer": 9, "max_source_mass": 0.3108460605144501, "mixer_index": 17, "normalized_entropy": 0.9669176780018418, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.2880150079727173, "median": 0.27969539165496826, "p05": 0.18933802843093872, "p95": 0.45879441499710083 }, { "label": "completed_group_1", "mean": 0.16260020434856415, "median": 0.16053099930286407, "p05": 0.11509333550930023, "p95": 0.22219550609588623 }, { "label": "completed_group_2", "mean": 0.2385387420654297, "median": 0.24041923880577087, "p05": 0.1582493633031845, "p95": 0.2989943325519562 }, { "label": "current_group_partial", "mean": 0.3108460605144501, "median": 0.3127351701259613, "p05": 0.2580314874649048, "p95": 0.3545264005661011 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3765960931777954, "group": 3, "latest_source_mass": 0.24758204817771912, "layer": 10, "max_source_mass": 0.28703436255455017, "mixer_index": 18, "normalized_entropy": 0.9930041784673634, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.28703436255455017, "median": 0.2873149514198303, "p05": 0.2421720325946808, "p95": 0.32964712381362915 }, { "label": "completed_group_1", "mean": 0.2552259564399719, "median": 0.2551601231098175, "p05": 0.2324453592300415, "p95": 0.27810582518577576 }, { "label": "completed_group_2", "mean": 0.2101576328277588, "median": 0.20999819040298462, "p05": 0.17491281032562256, "p95": 0.24711720645427704 }, { "label": "current_group_partial", "mean": 0.24758204817771912, "median": 0.2449823021888733, "p05": 0.21694661676883698, "p95": 0.2901700437068939 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.3274900913238525, "group": 3, "latest_source_mass": 0.26038026809692383, "layer": 10, "max_source_mass": 0.33539247512817383, "mixer_index": 19, "normalized_entropy": 0.9575816857910796, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.33539247512817383, "median": 0.3373180031776428, "p05": 0.16021503508090973, "p95": 0.5567492246627808 }, { "label": "completed_group_1", "mean": 0.1987970769405365, "median": 0.19616791605949402, "p05": 0.13216234743595123, "p95": 0.2684130072593689 }, { "label": "completed_group_2", "mean": 0.20543015003204346, "median": 0.20896214246749878, "p05": 0.11321981251239777, "p95": 0.2748231887817383 }, { "label": "current_group_partial", "mean": 0.26038026809692383, "median": 0.2554967999458313, "p05": 0.1821153312921524, "p95": 0.3462901711463928 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3747133016586304, "group": 3, "latest_source_mass": 0.2833673655986786, "layer": 11, "max_source_mass": 0.2833673655986786, "mixer_index": 20, "normalized_entropy": 0.9916460314734998, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.27988842129707336, "median": 0.28038889169692993, "p05": 0.24265244603157043, "p95": 0.31478387117385864 }, { "label": "completed_group_1", "mean": 0.2360810488462448, "median": 0.2335505485534668, "p05": 0.21358957886695862, "p95": 0.2655458450317383 }, { "label": "completed_group_2", "mean": 0.20066317915916443, "median": 0.2006566822528839, "p05": 0.18401089310646057, "p95": 0.21715889871120453 }, { "label": "current_group_partial", "mean": 0.2833673655986786, "median": 0.28357356786727905, "p05": 0.2585292458534241, "p95": 0.30662211775779724 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.3239198923110962, "group": 3, "latest_source_mass": 0.2723969519138336, "layer": 11, "max_source_mass": 0.3361120820045471, "mixer_index": 21, "normalized_entropy": 0.9550063315857346, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.3361120820045471, "median": 0.3591133654117584, "p05": 0.14645838737487793, "p95": 0.49824467301368713 }, { "label": "completed_group_1", "mean": 0.20296622812747955, "median": 0.19982439279556274, "p05": 0.13988786935806274, "p95": 0.27185869216918945 }, { "label": "completed_group_2", "mean": 0.1885247528553009, "median": 0.18602871894836426, "p05": 0.12637801468372345, "p95": 0.2539179027080536 }, { "label": "current_group_partial", "mean": 0.2723969519138336, "median": 0.25681596994400024, "p05": 0.19296783208847046, "p95": 0.3959115445613861 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3692121505737305, "group": 3, "latest_source_mass": 0.2974509000778198, "layer": 12, "max_source_mass": 0.2974509000778198, "mixer_index": 22, "normalized_entropy": 0.9876777897788168, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.28167375922203064, "median": 0.27825769782066345, "p05": 0.23325958847999573, "p95": 0.3442809581756592 }, { "label": "completed_group_1", "mean": 0.21946553885936737, "median": 0.21849599480628967, "p05": 0.18336281180381775, "p95": 0.2548777461051941 }, { "label": "completed_group_2", "mean": 0.20140981674194336, "median": 0.2013496309518814, "p05": 0.1822294294834137, "p95": 0.2219773232936859 }, { "label": "current_group_partial", "mean": 0.2974509000778198, "median": 0.297912061214447, "p05": 0.26495662331581116, "p95": 0.32783159613609314 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.3342361450195312, "group": 3, "latest_source_mass": 0.23993611335754395, "layer": 12, "max_source_mass": 0.36468517780303955, "mixer_index": 23, "normalized_entropy": 0.9624479348972428, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.36468517780303955, "median": 0.36720338463783264, "p05": 0.24960748851299286, "p95": 0.4882795810699463 }, { "label": "completed_group_1", "mean": 0.20593920350074768, "median": 0.2046196460723877, "p05": 0.16298335790634155, "p95": 0.25410282611846924 }, { "label": "completed_group_2", "mean": 0.18943950533866882, "median": 0.1881066858768463, "p05": 0.13994796574115753, "p95": 0.24192847311496735 }, { "label": "current_group_partial", "mean": 0.23993611335754395, "median": 0.232496976852417, "p05": 0.17571662366390228, "p95": 0.320015549659729 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3645737171173096, "group": 4, "latest_source_mass": 0.30504196882247925, "layer": 13, "max_source_mass": 0.30504196882247925, "mixer_index": 24, "normalized_entropy": 0.9843318673062809, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.2597482204437256, "median": 0.2599886655807495, "p05": 0.23272822797298431, "p95": 0.2871353030204773 }, { "label": "completed_group_1", "mean": 0.22607669234275818, "median": 0.21320533752441406, "p05": 0.1705966293811798, "p95": 0.32829806208610535 }, { "label": "completed_group_2", "mean": 0.209133118391037, "median": 0.20912399888038635, "p05": 0.19090500473976135, "p95": 0.2265017032623291 }, { "label": "completed_group_3", "mean": 0.30504196882247925, "median": 0.3161492347717285, "p05": 0.20431193709373474, "p95": 0.37756454944610596 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.5910131931304932, "group": 4, "latest_source_mass": 0.19691801071166992, "layer": 13, "max_source_mass": 0.2617476284503937, "mixer_index": 25, "normalized_entropy": 0.988552078237214, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.2617476284503937, "median": 0.2677137851715088, "p05": 0.20086795091629028, "p95": 0.31516557931900024 }, { "label": "completed_group_1", "mean": 0.18299485743045807, "median": 0.1818893998861313, "p05": 0.1605435013771057, "p95": 0.20783139765262604 }, { "label": "completed_group_2", "mean": 0.17935845255851746, "median": 0.18049052357673645, "p05": 0.14385172724723816, "p95": 0.2110004425048828 }, { "label": "completed_group_3", "mean": 0.17898105084896088, "median": 0.17628446221351624, "p05": 0.14925207197666168, "p95": 0.21800483763217926 }, { "label": "current_group_partial", "mean": 0.19691801071166992, "median": 0.1964835375547409, "p05": 0.16946497559547424, "p95": 0.22626768052577972 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.5884685516357422, "group": 4, "latest_source_mass": 0.26269519329071045, "layer": 14, "max_source_mass": 0.26269519329071045, "mixer_index": 26, "normalized_entropy": 0.9869710035805953, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.2046908289194107, "median": 0.2052469551563263, "p05": 0.18031299114227295, "p95": 0.2295369803905487 }, { "label": "completed_group_1", "mean": 0.17294223606586456, "median": 0.16847725212574005, "p05": 0.15012723207473755, "p95": 0.21394622325897217 }, { "label": "completed_group_2", "mean": 0.15044090151786804, "median": 0.1493840515613556, "p05": 0.1370292603969574, "p95": 0.1660556048154831 }, { "label": "completed_group_3", "mean": 0.20923087000846863, "median": 0.20989257097244263, "p05": 0.17921577394008636, "p95": 0.23829039931297302 }, { "label": "current_group_partial", "mean": 0.26269519329071045, "median": 0.2656410038471222, "p05": 0.23202235996723175, "p95": 0.28119558095932007 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.599259853363037, "group": 4, "latest_source_mass": 0.21076934039592743, "layer": 14, "max_source_mass": 0.21657392382621765, "mixer_index": 27, "normalized_entropy": 0.9936760163331371, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.21657392382621765, "median": 0.21635186672210693, "p05": 0.15601962804794312, "p95": 0.2927207350730896 }, { "label": "completed_group_1", "mean": 0.20580385625362396, "median": 0.20571252703666687, "p05": 0.18009206652641296, "p95": 0.2311524748802185 }, { "label": "completed_group_2", "mean": 0.19366896152496338, "median": 0.194598987698555, "p05": 0.16221287846565247, "p95": 0.22272710502147675 }, { "label": "completed_group_3", "mean": 0.1731838881969452, "median": 0.17174121737480164, "p05": 0.14892742037773132, "p95": 0.19922053813934326 }, { "label": "current_group_partial", "mean": 0.21076934039592743, "median": 0.20835596323013306, "p05": 0.1771993488073349, "p95": 0.2505151629447937 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.5786192417144775, "group": 4, "latest_source_mass": 0.2778070569038391, "layer": 15, "max_source_mass": 0.2778070569038391, "mixer_index": 28, "normalized_entropy": 0.980851283245209, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.20599736273288727, "median": 0.20488622784614563, "p05": 0.17115657031536102, "p95": 0.25322195887565613 }, { "label": "completed_group_1", "mean": 0.20593789219856262, "median": 0.20346379280090332, "p05": 0.18622605502605438, "p95": 0.2330799102783203 }, { "label": "completed_group_2", "mean": 0.13473063707351685, "median": 0.13439436256885529, "p05": 0.11930938065052032, "p95": 0.15121281147003174 }, { "label": "completed_group_3", "mean": 0.17552703619003296, "median": 0.1757315993309021, "p05": 0.1576612889766693, "p95": 0.1928577572107315 }, { "label": "current_group_partial", "mean": 0.2778070569038391, "median": 0.2790508270263672, "p05": 0.24338077008724213, "p95": 0.3088063597679138 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.580681324005127, "group": 4, "latest_source_mass": 0.2450503408908844, "layer": 15, "max_source_mass": 0.2450503408908844, "mixer_index": 29, "normalized_entropy": 0.9821325270103262, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.21277281641960144, "median": 0.20640528202056885, "p05": 0.12015408277511597, "p95": 0.3659750819206238 }, { "label": "completed_group_1", "mean": 0.18522509932518005, "median": 0.1863747239112854, "p05": 0.1475280076265335, "p95": 0.21663622558116913 }, { "label": "completed_group_2", "mean": 0.18489867448806763, "median": 0.1888006329536438, "p05": 0.12606219947338104, "p95": 0.22459235787391663 }, { "label": "completed_group_3", "mean": 0.17205308377742767, "median": 0.17103317379951477, "p05": 0.13283535838127136, "p95": 0.20961183309555054 }, { "label": "current_group_partial", "mean": 0.2450503408908844, "median": 0.24511924386024475, "p05": 0.19535338878631592, "p95": 0.2939102053642273 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.5705549716949463, "group": 4, "latest_source_mass": 0.28460633754730225, "layer": 16, "max_source_mass": 0.28460633754730225, "mixer_index": 30, "normalized_entropy": 0.9758406705603525, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.2125052809715271, "median": 0.21049857139587402, "p05": 0.16735027730464935, "p95": 0.2640833854675293 }, { "label": "completed_group_1", "mean": 0.20224158465862274, "median": 0.20027190446853638, "p05": 0.16782984137535095, "p95": 0.24093782901763916 }, { "label": "completed_group_2", "mean": 0.12915468215942383, "median": 0.12795840203762054, "p05": 0.1122070699930191, "p95": 0.14902588725090027 }, { "label": "completed_group_3", "mean": 0.1714920997619629, "median": 0.1704360991716385, "p05": 0.15070456266403198, "p95": 0.19576270878314972 }, { "label": "current_group_partial", "mean": 0.28460633754730225, "median": 0.28339242935180664, "p05": 0.2418026626110077, "p95": 0.33516958355903625 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.5599372386932373, "group": 4, "latest_source_mass": 0.2475198209285736, "layer": 16, "max_source_mass": 0.2665032148361206, "mixer_index": 31, "normalized_entropy": 0.9692435021205642, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.2665032148361206, "median": 0.2523844242095947, "p05": 0.16442818939685822, "p95": 0.41760408878326416 }, { "label": "completed_group_1", "mean": 0.181569442152977, "median": 0.18072238564491272, "p05": 0.1432785987854004, "p95": 0.2251572459936142 }, { "label": "completed_group_2", "mean": 0.15844660997390747, "median": 0.15800738334655762, "p05": 0.11516115069389343, "p95": 0.20221972465515137 }, { "label": "completed_group_3", "mean": 0.14596089720726013, "median": 0.14446985721588135, "p05": 0.10661658644676208, "p95": 0.18743650615215302 }, { "label": "current_group_partial", "mean": 0.2475198209285736, "median": 0.24823713302612305, "p05": 0.1857534945011139, "p95": 0.30610787868499756 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.562828540802002, "group": 5, "latest_source_mass": 0.26428917050361633, "layer": 17, "max_source_mass": 0.26428917050361633, "mixer_index": 32, "normalized_entropy": 0.9710399691271056, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.21806953847408295, "median": 0.21765722334384918, "p05": 0.1832873821258545, "p95": 0.256405234336853 }, { "label": "completed_group_1", "mean": 0.22324788570404053, "median": 0.2111721932888031, "p05": 0.1587892323732376, "p95": 0.33203551173210144 }, { "label": "completed_group_2", "mean": 0.11591021716594696, "median": 0.11326268315315247, "p05": 0.09832467138767242, "p95": 0.13705819845199585 }, { "label": "completed_group_3", "mean": 0.17848318815231323, "median": 0.1790149211883545, "p05": 0.1343652456998825, "p95": 0.21781212091445923 }, { "label": "completed_group_4", "mean": 0.26428917050361633, "median": 0.27161818742752075, "p05": 0.18240834772586823, "p95": 0.3144339919090271 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.761409044265747, "group": 5, "latest_source_mass": 0.14106416702270508, "layer": 17, "max_source_mass": 0.20728391408920288, "mixer_index": 33, "normalized_entropy": 0.9830611053081897, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1612042486667633, "median": 0.14052782952785492, "p05": 0.09785228967666626, "p95": 0.30037230253219604 }, { "label": "completed_group_1", "mean": 0.16519047319889069, "median": 0.16112321615219116, "p05": 0.13238948583602905, "p95": 0.2119981199502945 }, { "label": "completed_group_2", "mean": 0.17529632151126862, "median": 0.18006199598312378, "p05": 0.12263078987598419, "p95": 0.20985063910484314 }, { "label": "completed_group_3", "mean": 0.14996090531349182, "median": 0.1468663513660431, "p05": 0.11909312009811401, "p95": 0.18974041938781738 }, { "label": "completed_group_4", "mean": 0.20728391408920288, "median": 0.2042727917432785, "p05": 0.1449805498123169, "p95": 0.27828457951545715 }, { "label": "current_group_partial", "mean": 0.14106416702270508, "median": 0.1405012309551239, "p05": 0.10205969214439392, "p95": 0.1835198849439621 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.7366840839385986, "group": 5, "latest_source_mass": 0.2530929446220398, "layer": 18, "max_source_mass": 0.2530929446220398, "mixer_index": 34, "normalized_entropy": 0.9692618422085502, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.18514017760753632, "median": 0.18704235553741455, "p05": 0.13617590069770813, "p95": 0.22916007041931152 }, { "label": "completed_group_1", "mean": 0.1663871705532074, "median": 0.16571956872940063, "p05": 0.1462152749300003, "p95": 0.18810859322547913 }, { "label": "completed_group_2", "mean": 0.09063515067100525, "median": 0.08811916410923004, "p05": 0.07649004459381104, "p95": 0.11148498952388763 }, { "label": "completed_group_3", "mean": 0.11971023678779602, "median": 0.11747527122497559, "p05": 0.10390099883079529, "p95": 0.1423199474811554 }, { "label": "completed_group_4", "mean": 0.18503433465957642, "median": 0.18477493524551392, "p05": 0.15245835483074188, "p95": 0.22021599113941193 }, { "label": "current_group_partial", "mean": 0.2530929446220398, "median": 0.25188589096069336, "p05": 0.22554033994674683, "p95": 0.2833186984062195 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.777672290802002, "group": 5, "latest_source_mass": 0.19777557253837585, "layer": 18, "max_source_mass": 0.19777557253837585, "mixer_index": 35, "normalized_entropy": 0.9921377960222963, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.15483248233795166, "median": 0.14712893962860107, "p05": 0.12276758253574371, "p95": 0.22418904304504395 }, { "label": "completed_group_1", "mean": 0.16576308012008667, "median": 0.16587528586387634, "p05": 0.13462601602077484, "p95": 0.19553470611572266 }, { "label": "completed_group_2", "mean": 0.16957050561904907, "median": 0.17099428176879883, "p05": 0.14377880096435547, "p95": 0.19032810628414154 }, { "label": "completed_group_3", "mean": 0.14225168526172638, "median": 0.14081379771232605, "p05": 0.12333962321281433, "p95": 0.1642245650291443 }, { "label": "completed_group_4", "mean": 0.16980668902397156, "median": 0.16821962594985962, "p05": 0.13770604133605957, "p95": 0.20901428163051605 }, { "label": "current_group_partial", "mean": 0.19777557253837585, "median": 0.19705063104629517, "p05": 0.1461065709590912, "p95": 0.2508929371833801 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.7414222955703735, "group": 5, "latest_source_mass": 0.25494134426116943, "layer": 19, "max_source_mass": 0.25494134426116943, "mixer_index": 36, "normalized_entropy": 0.9719062884710925, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.18020270764827728, "median": 0.180809885263443, "p05": 0.1366642266511917, "p95": 0.22788876295089722 }, { "label": "completed_group_1", "mean": 0.1900298148393631, "median": 0.1896573156118393, "p05": 0.17211011052131653, "p95": 0.20767614245414734 }, { "label": "completed_group_2", "mean": 0.09944289177656174, "median": 0.09800659120082855, "p05": 0.08568179607391357, "p95": 0.11607980728149414 }, { "label": "completed_group_3", "mean": 0.11960913985967636, "median": 0.11915834993124008, "p05": 0.10471149533987045, "p95": 0.1360406130552292 }, { "label": "completed_group_4", "mean": 0.1557741016149521, "median": 0.15596863627433777, "p05": 0.13432671129703522, "p95": 0.1770717203617096 }, { "label": "current_group_partial", "mean": 0.25494134426116943, "median": 0.25551509857177734, "p05": 0.22669723629951477, "p95": 0.2819991707801819 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.733978509902954, "group": 5, "latest_source_mass": 0.2618475556373596, "layer": 19, "max_source_mass": 0.2618475556373596, "mixer_index": 37, "normalized_entropy": 0.9677518325883359, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.15204599499702454, "median": 0.1308840811252594, "p05": 0.09369519352912903, "p95": 0.32571133971214294 }, { "label": "completed_group_1", "mean": 0.14166373014450073, "median": 0.14088967442512512, "p05": 0.11298352479934692, "p95": 0.17192427814006805 }, { "label": "completed_group_2", "mean": 0.1616208553314209, "median": 0.1641538441181183, "p05": 0.11296901851892471, "p95": 0.1948811560869217 }, { "label": "completed_group_3", "mean": 0.1256796270608902, "median": 0.122653529047966, "p05": 0.09500046819448471, "p95": 0.16056732833385468 }, { "label": "completed_group_4", "mean": 0.1571422517299652, "median": 0.1528569459915161, "p05": 0.11623523384332657, "p95": 0.21299627423286438 }, { "label": "current_group_partial", "mean": 0.2618475556373596, "median": 0.26793089509010315, "p05": 0.15509119629859924, "p95": 0.339982271194458 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.7344707250595093, "group": 5, "latest_source_mass": 0.2615981698036194, "layer": 20, "max_source_mass": 0.2615981698036194, "mixer_index": 38, "normalized_entropy": 0.9680265430977588, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1829337775707245, "median": 0.183739572763443, "p05": 0.13839715719223022, "p95": 0.2284226417541504 }, { "label": "completed_group_1", "mean": 0.18416839838027954, "median": 0.18364182114601135, "p05": 0.16665373742580414, "p95": 0.20437687635421753 }, { "label": "completed_group_2", "mean": 0.09014187008142471, "median": 0.08851997554302216, "p05": 0.07715612649917603, "p95": 0.10642829537391663 }, { "label": "completed_group_3", "mean": 0.12335794419050217, "median": 0.12144016474485397, "p05": 0.10556001961231232, "p95": 0.14631223678588867 }, { "label": "completed_group_4", "mean": 0.15779981017112732, "median": 0.15796968340873718, "p05": 0.1340107023715973, "p95": 0.18188069760799408 }, { "label": "current_group_partial", "mean": 0.2615981698036194, "median": 0.262384295463562, "p05": 0.230574831366539, "p95": 0.28982529044151306 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.735896348953247, "group": 5, "latest_source_mass": 0.27010154724121094, "layer": 20, "max_source_mass": 0.27010154724121094, "mixer_index": 39, "normalized_entropy": 0.9688221989423192, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.17276549339294434, "median": 0.16886113584041595, "p05": 0.10281073302030563, "p95": 0.2923668622970581 }, { "label": "completed_group_1", "mean": 0.15101772546768188, "median": 0.14904022216796875, "p05": 0.13034172356128693, "p95": 0.17705516517162323 }, { "label": "completed_group_2", "mean": 0.13220664858818054, "median": 0.1315174400806427, "p05": 0.10168404132127762, "p95": 0.16267849504947662 }, { "label": "completed_group_3", "mean": 0.11932596564292908, "median": 0.11731794476509094, "p05": 0.09613680094480515, "p95": 0.1465223878622055 }, { "label": "completed_group_4", "mean": 0.15458261966705322, "median": 0.15350666642189026, "p05": 0.12296099960803986, "p95": 0.19313591718673706 }, { "label": "current_group_partial", "mean": 0.27010154724121094, "median": 0.27393096685409546, "p05": 0.19484001398086548, "p95": 0.3264923095703125 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.7644665241241455, "group": 6, "latest_source_mass": 0.2094498574733734, "layer": 21, "max_source_mass": 0.2094498574733734, "mixer_index": 40, "normalized_entropy": 0.9847675173076283, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.19160009920597076, "median": 0.196849524974823, "p05": 0.12982682883739471, "p95": 0.24405300617218018 }, { "label": "completed_group_1", "mean": 0.1915181279182434, "median": 0.1907939314842224, "p05": 0.17317205667495728, "p95": 0.2111247181892395 }, { "label": "completed_group_2", "mean": 0.11520625650882721, "median": 0.11408726871013641, "p05": 0.10055835545063019, "p95": 0.13332504034042358 }, { "label": "completed_group_3", "mean": 0.1341419219970703, "median": 0.13221079111099243, "p05": 0.11641807854175568, "p95": 0.15773889422416687 }, { "label": "completed_group_4", "mean": 0.1580837219953537, "median": 0.15754032135009766, "p05": 0.13662610948085785, "p95": 0.18392926454544067 }, { "label": "completed_group_5", "mean": 0.2094498574733734, "median": 0.20958366990089417, "p05": 0.1805649697780609, "p95": 0.23726987838745117 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.6835086345672607, "group": 6, "latest_source_mass": 0.43279585242271423, "layer": 21, "max_source_mass": 0.43279585242271423, "mixer_index": 41, "normalized_entropy": 0.8651522966692777, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.10065847635269165, "median": 0.09005473554134369, "p05": 0.05753578990697861, "p95": 0.19214311242103577 }, { "label": "completed_group_1", "mean": 0.09339895844459534, "median": 0.09201203286647797, "p05": 0.07428346574306488, "p95": 0.11678388714790344 }, { "label": "completed_group_2", "mean": 0.09682720899581909, "median": 0.09625889360904694, "p05": 0.07657647132873535, "p95": 0.11835116147994995 }, { "label": "completed_group_3", "mean": 0.08155890554189682, "median": 0.08032877743244171, "p05": 0.06701995432376862, "p95": 0.09893998503684998 }, { "label": "completed_group_4", "mean": 0.09126688539981842, "median": 0.09069330245256424, "p05": 0.07231517881155014, "p95": 0.1128317192196846 }, { "label": "completed_group_5", "mean": 0.10349369794130325, "median": 0.10402766615152359, "p05": 0.07336798310279846, "p95": 0.1318768411874771 }, { "label": "current_group_partial", "mean": 0.43279585242271423, "median": 0.4367910623550415, "p05": 0.3485929071903229, "p95": 0.5040479302406311 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.7937740087509155, "group": 6, "latest_source_mass": 0.3452628254890442, "layer": 22, "max_source_mass": 0.3452628254890442, "mixer_index": 42, "normalized_entropy": 0.9218174896830382, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.13723835349082947, "median": 0.1412133425474167, "p05": 0.08910952508449554, "p95": 0.17562061548233032 }, { "label": "completed_group_1", "mean": 0.12937891483306885, "median": 0.13051339983940125, "p05": 0.10541269928216934, "p95": 0.1487061083316803 }, { "label": "completed_group_2", "mean": 0.07450413703918457, "median": 0.07415075600147247, "p05": 0.06578464806079865, "p95": 0.08426731824874878 }, { "label": "completed_group_3", "mean": 0.08537043631076813, "median": 0.08443015813827515, "p05": 0.07344960421323776, "p95": 0.09956531971693039 }, { "label": "completed_group_4", "mean": 0.09880410879850388, "median": 0.09825065732002258, "p05": 0.08438277244567871, "p95": 0.11530908197164536 }, { "label": "completed_group_5", "mean": 0.12944120168685913, "median": 0.12968069314956665, "p05": 0.10858328640460968, "p95": 0.15059794485569 }, { "label": "current_group_partial", "mean": 0.3452628254890442, "median": 0.33866745233535767, "p05": 0.288330078125, "p95": 0.42612025141716003 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.5790722370147705, "group": 6, "latest_source_mass": 0.49195876717567444, "layer": 22, "max_source_mass": 0.49195876717567444, "mixer_index": 43, "normalized_entropy": 0.8114826050839847, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.09500400722026825, "median": 0.07923799753189087, "p05": 0.045484717935323715, "p95": 0.2259966880083084 }, { "label": "completed_group_1", "mean": 0.08892714232206345, "median": 0.0876917839050293, "p05": 0.06900187581777573, "p95": 0.11303731799125671 }, { "label": "completed_group_2", "mean": 0.09081116318702698, "median": 0.0898551493883133, "p05": 0.06699725985527039, "p95": 0.11749734729528427 }, { "label": "completed_group_3", "mean": 0.07538282871246338, "median": 0.07381957024335861, "p05": 0.05869410187005997, "p95": 0.09544046223163605 }, { "label": "completed_group_4", "mean": 0.07685283571481705, "median": 0.07580418884754181, "p05": 0.056799523532390594, "p95": 0.10076010227203369 }, { "label": "completed_group_5", "mean": 0.08106326311826706, "median": 0.08067160844802856, "p05": 0.05696551501750946, "p95": 0.10726240277290344 }, { "label": "current_group_partial", "mean": 0.49195876717567444, "median": 0.49396610260009766, "p05": 0.3897285759449005, "p95": 0.583945631980896 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.8034920692443848, "group": 6, "latest_source_mass": 0.33595213294029236, "layer": 23, "max_source_mass": 0.33595213294029236, "mixer_index": 44, "normalized_entropy": 0.926811584861681, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.142653226852417, "median": 0.14694267511367798, "p05": 0.08785572648048401, "p95": 0.18721188604831696 }, { "label": "completed_group_1", "mean": 0.13419847190380096, "median": 0.13438312709331512, "p05": 0.11451410502195358, "p95": 0.15392597019672394 }, { "label": "completed_group_2", "mean": 0.07437049597501755, "median": 0.07391499727964401, "p05": 0.06574448198080063, "p95": 0.08430380374193192 }, { "label": "completed_group_3", "mean": 0.09070361405611038, "median": 0.08887749910354614, "p05": 0.07508289068937302, "p95": 0.11117327958345413 }, { "label": "completed_group_4", "mean": 0.1013825386762619, "median": 0.10140319168567657, "p05": 0.08447366952896118, "p95": 0.11889617145061493 }, { "label": "completed_group_5", "mean": 0.12073953449726105, "median": 0.12033741921186447, "p05": 0.10022275149822235, "p95": 0.1415012627840042 }, { "label": "current_group_partial", "mean": 0.33595213294029236, "median": 0.330989271402359, "p05": 0.27680477499961853, "p95": 0.41842275857925415 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.5278494358062744, "group": 6, "latest_source_mass": 0.51820969581604, "layer": 23, "max_source_mass": 0.51820969581604, "mixer_index": 45, "normalized_entropy": 0.7851592924514033, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.10045768320560455, "median": 0.08486579358577728, "p05": 0.04896760731935501, "p95": 0.2206544280052185 }, { "label": "completed_group_1", "mean": 0.08844950795173645, "median": 0.08638796210289001, "p05": 0.06730383634567261, "p95": 0.11565113067626953 }, { "label": "completed_group_2", "mean": 0.08561251312494278, "median": 0.08505219221115112, "p05": 0.0625130906701088, "p95": 0.11068116128444672 }, { "label": "completed_group_3", "mean": 0.06598107516765594, "median": 0.06433478742837906, "p05": 0.04859655350446701, "p95": 0.08642670512199402 }, { "label": "completed_group_4", "mean": 0.06767018139362335, "median": 0.06678327918052673, "p05": 0.047009289264678955, "p95": 0.09129223972558975 }, { "label": "completed_group_5", "mean": 0.07361933588981628, "median": 0.07290077209472656, "p05": 0.049975596368312836, "p95": 0.09868258237838745 }, { "label": "current_group_partial", "mean": 0.51820969581604, "median": 0.5205609798431396, "p05": 0.40315571427345276, "p95": 0.6213710308074951 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.8435695171356201, "group": 6, "latest_source_mass": 0.29524505138397217, "layer": 24, "max_source_mass": 0.29524505138397217, "mixer_index": 46, "normalized_entropy": 0.9474073188993969, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.14839638769626617, "median": 0.15232303738594055, "p05": 0.08978340029716492, "p95": 0.19894656538963318 }, { "label": "completed_group_1", "mean": 0.1482572853565216, "median": 0.14857065677642822, "p05": 0.13154153525829315, "p95": 0.16430144011974335 }, { "label": "completed_group_2", "mean": 0.08113027364015579, "median": 0.08067925274372101, "p05": 0.07102686911821365, "p95": 0.09267604351043701 }, { "label": "completed_group_3", "mean": 0.09534022957086563, "median": 0.09320184588432312, "p05": 0.0778813511133194, "p95": 0.11805401742458344 }, { "label": "completed_group_4", "mean": 0.10612697899341583, "median": 0.10644470155239105, "p05": 0.08815888315439224, "p95": 0.12415381520986557 }, { "label": "completed_group_5", "mean": 0.12550383806228638, "median": 0.1254296898841858, "p05": 0.10509618371725082, "p95": 0.14616522192955017 }, { "label": "current_group_partial", "mean": 0.29524505138397217, "median": 0.29100558161735535, "p05": 0.24107839167118073, "p95": 0.37045177817344666 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.6421003341674805, "group": 6, "latest_source_mass": 0.4530346691608429, "layer": 24, "max_source_mass": 0.4530346691608429, "mixer_index": 47, "normalized_entropy": 0.8438726397334819, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.11422763019800186, "median": 0.09945468604564667, "p05": 0.06323201954364777, "p95": 0.23352834582328796 }, { "label": "completed_group_1", "mean": 0.10543527454137802, "median": 0.1044958084821701, "p05": 0.08478721231222153, "p95": 0.1284603774547577 }, { "label": "completed_group_2", "mean": 0.09285669028759003, "median": 0.09177995473146439, "p05": 0.06902404874563217, "p95": 0.12035554647445679 }, { "label": "completed_group_3", "mean": 0.07746599614620209, "median": 0.07653152942657471, "p05": 0.058040909469127655, "p95": 0.09812526404857635 }, { "label": "completed_group_4", "mean": 0.07571054995059967, "median": 0.07457360625267029, "p05": 0.05648855119943619, "p95": 0.0989091694355011 }, { "label": "completed_group_5", "mean": 0.08126918971538544, "median": 0.08049014210700989, "p05": 0.05931335315108299, "p95": 0.10596252232789993 }, { "label": "current_group_partial", "mean": 0.4530346691608429, "median": 0.455755352973938, "p05": 0.3413957357406616, "p95": 0.5519019365310669 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.8733552694320679, "group": 7, "latest_source_mass": 0.26451510190963745, "layer": 25, "max_source_mass": 0.26451510190963745, "mixer_index": 48, "normalized_entropy": 0.9627141676307774, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.14798463881015778, "median": 0.14657661318778992, "p05": 0.09472498297691345, "p95": 0.2258487045764923 }, { "label": "completed_group_1", "mean": 0.160526841878891, "median": 0.16084568202495575, "p05": 0.1358838975429535, "p95": 0.18423213064670563 }, { "label": "completed_group_2", "mean": 0.09668850898742676, "median": 0.09659601747989655, "p05": 0.08555105328559875, "p95": 0.1084243580698967 }, { "label": "completed_group_3", "mean": 0.10477185994386673, "median": 0.10316876322031021, "p05": 0.08876212686300278, "p95": 0.12562818825244904 }, { "label": "completed_group_4", "mean": 0.10857057571411133, "median": 0.1074000895023346, "p05": 0.09068692475557327, "p95": 0.12911471724510193 }, { "label": "completed_group_5", "mean": 0.11694245785474777, "median": 0.11679476499557495, "p05": 0.0976894274353981, "p95": 0.13528725504875183 }, { "label": "completed_group_6", "mean": 0.26451510190963745, "median": 0.26442092657089233, "p05": 0.21035388112068176, "p95": 0.31906524300575256 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.784506916999817, "group": 7, "latest_source_mass": 0.27427971363067627, "layer": 25, "max_source_mass": 0.29875272512435913, "mixer_index": 49, "normalized_entropy": 0.8581664265292297, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.08589775860309601, "median": 0.07514797151088715, "p05": 0.03144541755318642, "p95": 0.20497030019760132 }, { "label": "completed_group_1", "mean": 0.08786046504974365, "median": 0.08646859228610992, "p05": 0.06123892217874527, "p95": 0.11937689781188965 }, { "label": "completed_group_2", "mean": 0.07714234292507172, "median": 0.07644426077604294, "p05": 0.05296650156378746, "p95": 0.10448302328586578 }, { "label": "completed_group_3", "mean": 0.06314892321825027, "median": 0.06132982298731804, "p05": 0.044174306094646454, "p95": 0.08466912060976028 }, { "label": "completed_group_4", "mean": 0.05684571713209152, "median": 0.055630408227443695, "p05": 0.036537639796733856, "p95": 0.08176163583993912 }, { "label": "completed_group_5", "mean": 0.056072384119033813, "median": 0.0549161471426487, "p05": 0.03825453296303749, "p95": 0.07769857347011566 }, { "label": "completed_group_6", "mean": 0.29875272512435913, "median": 0.29761478304862976, "p05": 0.17369259893894196, "p95": 0.4214131534099579 }, { "label": "current_group_partial", "mean": 0.27427971363067627, "median": 0.26517564058303833, "p05": 0.11691330373287201, "p95": 0.45670589804649353 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.9675533771514893, "group": 7, "latest_source_mass": 0.24859297275543213, "layer": 26, "max_source_mass": 0.24859297275543213, "mixer_index": 50, "normalized_entropy": 0.9461931666335953, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.11549945175647736, "median": 0.1186310350894928, "p05": 0.0548042394220829, "p95": 0.17459861934185028 }, { "label": "completed_group_1", "mean": 0.12592050433158875, "median": 0.12912830710411072, "p05": 0.08620737493038177, "p95": 0.15710318088531494 }, { "label": "completed_group_2", "mean": 0.07530814409255981, "median": 0.07495854794979095, "p05": 0.06174474209547043, "p95": 0.0894969180226326 }, { "label": "completed_group_3", "mean": 0.08277833461761475, "median": 0.08158879727125168, "p05": 0.06734651327133179, "p95": 0.10064717382192612 }, { "label": "completed_group_4", "mean": 0.08296641707420349, "median": 0.08213280141353607, "p05": 0.06705781072378159, "p95": 0.10095006227493286 }, { "label": "completed_group_5", "mean": 0.09016789495944977, "median": 0.08980530500411987, "p05": 0.07541801035404205, "p95": 0.10646115988492966 }, { "label": "completed_group_6", "mean": 0.17876628041267395, "median": 0.17772917449474335, "p05": 0.13528534770011902, "p95": 0.2264900654554367 }, { "label": "current_group_partial", "mean": 0.24859297275543213, "median": 0.245039701461792, "p05": 0.14620795845985413, "p95": 0.3620130121707916 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.7119876146316528, "group": 7, "latest_source_mass": 0.3658455014228821, "layer": 26, "max_source_mass": 0.3658455014228821, "mixer_index": 51, "normalized_entropy": 0.8232920138974705, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.08804735541343689, "median": 0.08122549951076508, "p05": 0.054039862006902695, "p95": 0.1443495899438858 }, { "label": "completed_group_1", "mean": 0.07527871429920197, "median": 0.0738755539059639, "p05": 0.05371527001261711, "p95": 0.10149700939655304 }, { "label": "completed_group_2", "mean": 0.05666808784008026, "median": 0.05508599430322647, "p05": 0.03937675803899765, "p95": 0.07947445660829544 }, { "label": "completed_group_3", "mean": 0.050419021397829056, "median": 0.04877372086048126, "p05": 0.03257015347480774, "p95": 0.0718536451458931 }, { "label": "completed_group_4", "mean": 0.05142981931567192, "median": 0.0498797781765461, "p05": 0.034862153232097626, "p95": 0.07353642582893372 }, { "label": "completed_group_5", "mean": 0.05313318967819214, "median": 0.051505643874406815, "p05": 0.037266045808792114, "p95": 0.07490143924951553 }, { "label": "completed_group_6", "mean": 0.2591783106327057, "median": 0.2531449794769287, "p05": 0.15121260285377502, "p95": 0.386227548122406 }, { "label": "current_group_partial", "mean": 0.3658455014228821, "median": 0.3656759262084961, "p05": 0.19218485057353973, "p95": 0.5332037210464478 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.9094492197036743, "group": 7, "latest_source_mass": 0.2766643464565277, "layer": 27, "max_source_mass": 0.2766643464565277, "mixer_index": 52, "normalized_entropy": 0.918250973365264, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.10337945818901062, "median": 0.10516610741615295, "p05": 0.05207033082842827, "p95": 0.1573200225830078 }, { "label": "completed_group_1", "mean": 0.10583682358264923, "median": 0.10595200955867767, "p05": 0.0643514022231102, "p95": 0.14548130333423615 }, { "label": "completed_group_2", "mean": 0.08642244338989258, "median": 0.08549235016107559, "p05": 0.06607823818922043, "p95": 0.11075147986412048 }, { "label": "completed_group_3", "mean": 0.08043928444385529, "median": 0.07886679470539093, "p05": 0.05936145409941673, "p95": 0.10334464907646179 }, { "label": "completed_group_4", "mean": 0.0717030018568039, "median": 0.07051880657672882, "p05": 0.05472247302532196, "p95": 0.09299077093601227 }, { "label": "completed_group_5", "mean": 0.0645938515663147, "median": 0.06373226642608643, "p05": 0.04421529173851013, "p95": 0.08712958544492722 }, { "label": "completed_group_6", "mean": 0.21096080541610718, "median": 0.20807480812072754, "p05": 0.14586198329925537, "p95": 0.2868158519268036 }, { "label": "current_group_partial", "mean": 0.2766643464565277, "median": 0.27722808718681335, "p05": 0.16094130277633667, "p95": 0.3963583707809448 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.7860023975372314, "group": 7, "latest_source_mass": 0.3244447112083435, "layer": 27, "max_source_mass": 0.3244447112083435, "mixer_index": 53, "normalized_entropy": 0.8588856006475877, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.09247160702943802, "median": 0.08733023703098297, "p05": 0.04943867400288582, "p95": 0.16303153336048126 }, { "label": "completed_group_1", "mean": 0.0784597396850586, "median": 0.07711629569530487, "p05": 0.05584436655044556, "p95": 0.10506026446819305 }, { "label": "completed_group_2", "mean": 0.07236532866954803, "median": 0.07097338140010834, "p05": 0.050456635653972626, "p95": 0.09917968511581421 }, { "label": "completed_group_3", "mean": 0.05986309051513672, "median": 0.057845089584589005, "p05": 0.04075033217668533, "p95": 0.08416998386383057 }, { "label": "completed_group_4", "mean": 0.05389959365129471, "median": 0.05301224812865257, "p05": 0.03848717734217644, "p95": 0.07268592715263367 }, { "label": "completed_group_5", "mean": 0.06059400737285614, "median": 0.05963316559791565, "p05": 0.04356392100453377, "p95": 0.08111730217933655 }, { "label": "completed_group_6", "mean": 0.2579019069671631, "median": 0.2530352473258972, "p05": 0.16421708464622498, "p95": 0.3691956102848053 }, { "label": "current_group_partial", "mean": 0.3244447112083435, "median": 0.3252047896385193, "p05": 0.17263495922088623, "p95": 0.47815465927124023 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.9232951402664185, "group": 7, "latest_source_mass": 0.2509572505950928, "layer": 28, "max_source_mass": 0.2509572505950928, "mixer_index": 54, "normalized_entropy": 0.9249094536760685, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.1160542219877243, "median": 0.1222282201051712, "p05": 0.061863381415605545, "p95": 0.16598135232925415 }, { "label": "completed_group_1", "mean": 0.11346358060836792, "median": 0.11177559196949005, "p05": 0.05926967412233353, "p95": 0.16970883309841156 }, { "label": "completed_group_2", "mean": 0.08816123753786087, "median": 0.08707751333713531, "p05": 0.06823235750198364, "p95": 0.1114775538444519 }, { "label": "completed_group_3", "mean": 0.08138670027256012, "median": 0.0797908753156662, "p05": 0.06149819865822792, "p95": 0.10523255914449692 }, { "label": "completed_group_4", "mean": 0.07161372900009155, "median": 0.07095701992511749, "p05": 0.055086471140384674, "p95": 0.09069444239139557 }, { "label": "completed_group_5", "mean": 0.06013647839426994, "median": 0.05960702896118164, "p05": 0.04596705362200737, "p95": 0.076435886323452 }, { "label": "completed_group_6", "mean": 0.21822679042816162, "median": 0.21632584929466248, "p05": 0.14715570211410522, "p95": 0.2940847873687744 }, { "label": "current_group_partial", "mean": 0.2509572505950928, "median": 0.24700899422168732, "p05": 0.14031530916690826, "p95": 0.37019768357276917 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.8294003009796143, "group": 7, "latest_source_mass": 0.20826560258865356, "layer": 28, "max_source_mass": 0.31431835889816284, "mixer_index": 55, "normalized_entropy": 0.879755580674689, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.08831655979156494, "median": 0.06431648135185242, "p05": 0.03252303972840309, "p95": 0.2706584930419922 }, { "label": "completed_group_1", "mean": 0.10072161257266998, "median": 0.1009141132235527, "p05": 0.06054181233048439, "p95": 0.14067479968070984 }, { "label": "completed_group_2", "mean": 0.0818098932504654, "median": 0.08032949268817902, "p05": 0.054586175829172134, "p95": 0.1145491972565651 }, { "label": "completed_group_3", "mean": 0.07514279335737228, "median": 0.07337591052055359, "p05": 0.04998338222503662, "p95": 0.10431315004825592 }, { "label": "completed_group_4", "mean": 0.06651485711336136, "median": 0.06524930149316788, "p05": 0.04414192587137222, "p95": 0.09315875917673111 }, { "label": "completed_group_5", "mean": 0.06491032987833023, "median": 0.06368150562047958, "p05": 0.04332352057099342, "p95": 0.09065432846546173 }, { "label": "completed_group_6", "mean": 0.31431835889816284, "median": 0.3147735297679901, "p05": 0.18699732422828674, "p95": 0.44058555364608765 }, { "label": "current_group_partial", "mean": 0.20826560258865356, "median": 0.1914007067680359, "p05": 0.08225662261247635, "p95": 0.38793477416038513 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.9393219947814941, "group": 8, "latest_source_mass": 0.15344023704528809, "layer": 29, "max_source_mass": 0.2604207992553711, "mixer_index": 56, "normalized_entropy": 0.9326167415193847, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.11041727662086487, "median": 0.10045937448740005, "p05": 0.024797730147838593, "p95": 0.279465913772583 }, { "label": "completed_group_1", "mean": 0.12080886960029602, "median": 0.12561748921871185, "p05": 0.0680447518825531, "p95": 0.16192926466464996 }, { "label": "completed_group_2", "mean": 0.09995828568935394, "median": 0.09907965362071991, "p05": 0.07490158826112747, "p95": 0.1276247799396515 }, { "label": "completed_group_3", "mean": 0.09934850037097931, "median": 0.09762611985206604, "p05": 0.06941570341587067, "p95": 0.134530171751976 }, { "label": "completed_group_4", "mean": 0.08031868189573288, "median": 0.07758662104606628, "p05": 0.05694210156798363, "p95": 0.11248748749494553 }, { "label": "completed_group_5", "mean": 0.075287364423275, "median": 0.07397153228521347, "p05": 0.05265223979949951, "p95": 0.10342685133218765 }, { "label": "completed_group_6", "mean": 0.2604207992553711, "median": 0.2616186738014221, "p05": 0.16666486859321594, "p95": 0.34887272119522095 }, { "label": "completed_group_7", "mean": 0.15344023704528809, "median": 0.13995957374572754, "p05": 0.05727577582001686, "p95": 0.29449376463890076 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.9925593137741089, "group": 8, "latest_source_mass": 0.18462148308753967, "layer": 29, "max_source_mass": 0.2503892779350281, "mixer_index": 57, "normalized_entropy": 0.9068528243889232, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.08698172122240067, "median": 0.07842238247394562, "p05": 0.02795225754380226, "p95": 0.1876923143863678 }, { "label": "completed_group_1", "mean": 0.08857265114784241, "median": 0.08714614808559418, "p05": 0.06365615874528885, "p95": 0.11900493502616882 }, { "label": "completed_group_2", "mean": 0.07299254834651947, "median": 0.07189398258924484, "p05": 0.05586059391498566, "p95": 0.0933830514550209 }, { "label": "completed_group_3", "mean": 0.060478877276182175, "median": 0.05862775444984436, "p05": 0.04474102333188057, "p95": 0.07903359085321426 }, { "label": "completed_group_4", "mean": 0.054115522652864456, "median": 0.052696481347084045, "p05": 0.03536852449178696, "p95": 0.07748772948980331 }, { "label": "completed_group_5", "mean": 0.056821923702955246, "median": 0.05573669448494911, "p05": 0.0398341529071331, "p95": 0.07687154412269592 }, { "label": "completed_group_6", "mean": 0.2503892779350281, "median": 0.2475910186767578, "p05": 0.15396791696548462, "p95": 0.357724130153656 }, { "label": "completed_group_7", "mean": 0.14502598345279694, "median": 0.13307344913482666, "p05": 0.06183449178934097, "p95": 0.2673647403717041 }, { "label": "current_group_partial", "mean": 0.18462148308753967, "median": 0.17473986744880676, "p05": 0.08267536759376526, "p95": 0.3240666389465332 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.068819999694824, "group": 8, "latest_source_mass": 0.15494346618652344, "layer": 30, "max_source_mass": 0.21415996551513672, "mixer_index": 58, "normalized_entropy": 0.9415605582761754, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.09429103136062622, "median": 0.0943363606929779, "p05": 0.03544093295931816, "p95": 0.16596338152885437 }, { "label": "completed_group_1", "mean": 0.09028280526399612, "median": 0.090491384267807, "p05": 0.05409309267997742, "p95": 0.12526515126228333 }, { "label": "completed_group_2", "mean": 0.09188266098499298, "median": 0.09010649472475052, "p05": 0.06878700107336044, "p95": 0.12181076407432556 }, { "label": "completed_group_3", "mean": 0.0735517293214798, "median": 0.07178769260644913, "p05": 0.0548262856900692, "p95": 0.09550250321626663 }, { "label": "completed_group_4", "mean": 0.06746294349431992, "median": 0.06608930230140686, "p05": 0.050633110105991364, "p95": 0.08874846994876862 }, { "label": "completed_group_5", "mean": 0.060166649520397186, "median": 0.060052454471588135, "p05": 0.039401471614837646, "p95": 0.08220525085926056 }, { "label": "completed_group_6", "mean": 0.21415996551513672, "median": 0.2118481993675232, "p05": 0.15205052495002747, "p95": 0.2836942672729492 }, { "label": "completed_group_7", "mean": 0.15325874090194702, "median": 0.14970678091049194, "p05": 0.08526452630758286, "p95": 0.23864519596099854 }, { "label": "current_group_partial", "mean": 0.15494346618652344, "median": 0.1473558098077774, "p05": 0.0697651207447052, "p95": 0.2689872086048126 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.0139355659484863, "group": 8, "latest_source_mass": 0.1372738927602768, "layer": 30, "max_source_mass": 0.24272148311138153, "mixer_index": 59, "normalized_entropy": 0.916581576012616, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.0956040769815445, "median": 0.09956102818250656, "p05": 0.03933291882276535, "p95": 0.16697832942008972 }, { "label": "completed_group_1", "mean": 0.09831400960683823, "median": 0.09554082155227661, "p05": 0.05786377936601639, "p95": 0.14979025721549988 }, { "label": "completed_group_2", "mean": 0.0714857280254364, "median": 0.06985784322023392, "p05": 0.04893886297941208, "p95": 0.10043824464082718 }, { "label": "completed_group_3", "mean": 0.0677608773112297, "median": 0.06590595096349716, "p05": 0.044468365609645844, "p95": 0.09555256366729736 }, { "label": "completed_group_4", "mean": 0.05654233694076538, "median": 0.054802894592285156, "p05": 0.037319183349609375, "p95": 0.08175043016672134 }, { "label": "completed_group_5", "mean": 0.05388312786817551, "median": 0.05288306996226311, "p05": 0.036485109478235245, "p95": 0.07530424743890762 }, { "label": "completed_group_6", "mean": 0.24272148311138153, "median": 0.23883119225502014, "p05": 0.1493627429008484, "p95": 0.35197100043296814 }, { "label": "completed_group_7", "mean": 0.17641445994377136, "median": 0.17445214092731476, "p05": 0.0891575887799263, "p95": 0.276222825050354 }, { "label": "current_group_partial", "mean": 0.1372738927602768, "median": 0.12184794992208481, "p05": 0.05821039155125618, "p95": 0.2647005021572113 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.0349321365356445, "group": 8, "latest_source_mass": 0.13534162938594818, "layer": 31, "max_source_mass": 0.2261163890361786, "mixer_index": 60, "normalized_entropy": 0.9261375270991514, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.10256819427013397, "median": 0.10616284608840942, "p05": 0.019877789542078972, "p95": 0.19627508521080017 }, { "label": "completed_group_1", "mean": 0.0938093364238739, "median": 0.09388002753257751, "p05": 0.034650709480047226, "p95": 0.15219232439994812 }, { "label": "completed_group_2", "mean": 0.0912778377532959, "median": 0.08955617249011993, "p05": 0.06106609106063843, "p95": 0.1285511702299118 }, { "label": "completed_group_3", "mean": 0.08394120633602142, "median": 0.0820196121931076, "p05": 0.05809478089213371, "p95": 0.11390723288059235 }, { "label": "completed_group_4", "mean": 0.06161237880587578, "median": 0.059880051761865616, "p05": 0.044582173228263855, "p95": 0.08485961705446243 }, { "label": "completed_group_5", "mean": 0.053358763456344604, "median": 0.052702393382787704, "p05": 0.03554790839552879, "p95": 0.07320070266723633 }, { "label": "completed_group_6", "mean": 0.2261163890361786, "median": 0.22380158305168152, "p05": 0.1451866328716278, "p95": 0.3134670853614807 }, { "label": "completed_group_7", "mean": 0.15197426080703735, "median": 0.1527099758386612, "p05": 0.0639471560716629, "p95": 0.2459363490343094 }, { "label": "current_group_partial", "mean": 0.13534162938594818, "median": 0.11519233882427216, "p05": 0.04325530678033829, "p95": 0.2904844284057617 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.046657085418701, "group": 8, "latest_source_mass": 0.09365127235651016, "layer": 31, "max_source_mass": 0.2572976052761078, "mixer_index": 61, "normalized_entropy": 0.9314737813009277, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.08380690217018127, "median": 0.06641843914985657, "p05": 0.039923232048749924, "p95": 0.23694062232971191 }, { "label": "completed_group_1", "mean": 0.1217331737279892, "median": 0.12164830416440964, "p05": 0.08828366547822952, "p95": 0.1561325639486313 }, { "label": "completed_group_2", "mean": 0.10242430865764618, "median": 0.10224691033363342, "p05": 0.06860878318548203, "p95": 0.13651791214942932 }, { "label": "completed_group_3", "mean": 0.09225838631391525, "median": 0.09045033901929855, "p05": 0.0655013769865036, "p95": 0.1235189214348793 }, { "label": "completed_group_4", "mean": 0.06788390874862671, "median": 0.0664045438170433, "p05": 0.047665324062108994, "p95": 0.09357043355703354 }, { "label": "completed_group_5", "mean": 0.06336672604084015, "median": 0.06275692582130432, "p05": 0.044327184557914734, "p95": 0.08375845849514008 }, { "label": "completed_group_6", "mean": 0.2572976052761078, "median": 0.25880685448646545, "p05": 0.14942821860313416, "p95": 0.36036571860313416 }, { "label": "completed_group_7", "mean": 0.11757771670818329, "median": 0.10582176595926285, "p05": 0.05813705176115036, "p95": 0.21660257875919342 }, { "label": "current_group_partial", "mean": 0.09365127235651016, "median": 0.08298523724079132, "p05": 0.04494771361351013, "p95": 0.18065699934959412 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.094015121459961, "group": 8, "latest_source_mass": 0.09723767638206482, "layer": 32, "max_source_mass": 0.2018478661775589, "mixer_index": 62, "normalized_entropy": 0.9530273523513089, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.10284869372844696, "median": 0.09173205494880676, "p05": 0.03822793811559677, "p95": 0.22231025993824005 }, { "label": "completed_group_1", "mean": 0.13057854771614075, "median": 0.13079363107681274, "p05": 0.0711527168750763, "p95": 0.19180890917778015 }, { "label": "completed_group_2", "mean": 0.10152992606163025, "median": 0.09965524077415466, "p05": 0.07341106235980988, "p95": 0.13697651028633118 }, { "label": "completed_group_3", "mean": 0.10079469531774521, "median": 0.10007210820913315, "p05": 0.07479676604270935, "p95": 0.12990058958530426 }, { "label": "completed_group_4", "mean": 0.0796234980225563, "median": 0.07812212407588959, "p05": 0.05766180902719498, "p95": 0.10730832070112228 }, { "label": "completed_group_5", "mean": 0.07438632845878601, "median": 0.07310014963150024, "p05": 0.05309801548719406, "p95": 0.10082833468914032 }, { "label": "completed_group_6", "mean": 0.2018478661775589, "median": 0.20017066597938538, "p05": 0.13963642716407776, "p95": 0.2733708918094635 }, { "label": "completed_group_7", "mean": 0.1111527606844902, "median": 0.10317055881023407, "p05": 0.05416924133896828, "p95": 0.1888933926820755 }, { "label": "current_group_partial", "mean": 0.09723767638206482, "median": 0.08349506556987762, "p05": 0.03931424394249916, "p95": 0.1855989694595337 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.057201385498047, "group": 8, "latest_source_mass": 0.09480181336402893, "layer": 32, "max_source_mass": 0.24551761150360107, "mixer_index": 63, "normalized_entropy": 0.9362726990757002, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.09265108406543732, "median": 0.07600092887878418, "p05": 0.04942645877599716, "p95": 0.20456968247890472 }, { "label": "completed_group_1", "mean": 0.11481083929538727, "median": 0.1135297492146492, "p05": 0.08239293098449707, "p95": 0.15063267946243286 }, { "label": "completed_group_2", "mean": 0.09007447212934494, "median": 0.0894046500325203, "p05": 0.06493785232305527, "p95": 0.11702875047922134 }, { "label": "completed_group_3", "mean": 0.08496076613664627, "median": 0.08339293301105499, "p05": 0.06288711726665497, "p95": 0.11049411445856094 }, { "label": "completed_group_4", "mean": 0.06770835816860199, "median": 0.06692098081111908, "p05": 0.0488700196146965, "p95": 0.09013402462005615 }, { "label": "completed_group_5", "mean": 0.05925413966178894, "median": 0.058933183550834656, "p05": 0.04426975175738335, "p95": 0.07536415755748749 }, { "label": "completed_group_6", "mean": 0.24551761150360107, "median": 0.24382448196411133, "p05": 0.15310630202293396, "p95": 0.3403708338737488 }, { "label": "completed_group_7", "mean": 0.15022093057632446, "median": 0.14221730828285217, "p05": 0.0825236365199089, "p95": 0.24877692759037018 }, { "label": "current_group_partial", "mean": 0.09480181336402893, "median": 0.08117073774337769, "p05": 0.04724843427538872, "p95": 0.17635159194469452 } ], "sources": 9 }, { "branch": "output", "entropy_mean": 1.8435449600219727, "group": null, "latest_source_mass": 0.20717504620552063, "layer": null, "max_source_mass": 0.31774085760116577, "mixer_index": null, "normalized_entropy": 0.839033469331102, "offset": null, "source_summaries": [ { "label": "embedding", "mean": 0.1112452894449234, "median": 0.11452806740999222, "p05": 0.04704529047012329, "p95": 0.1625424325466156 }, { "label": "completed_group_1", "mean": 0.06898629665374756, "median": 0.06820575892925262, "p05": 0.04699244350194931, "p95": 0.09449291229248047 }, { "label": "completed_group_2", "mean": 0.05835210159420967, "median": 0.05662037432193756, "p05": 0.039408810436725616, "p95": 0.081703320145607 }, { "label": "completed_group_3", "mean": 0.042265649884939194, "median": 0.040285609662532806, "p05": 0.026811392977833748, "p95": 0.06341734528541565 }, { "label": "completed_group_4", "mean": 0.041034016758203506, "median": 0.039777740836143494, "p05": 0.027301941066980362, "p95": 0.05862141400575638 }, { "label": "completed_group_5", "mean": 0.04826357960700989, "median": 0.047516219317913055, "p05": 0.03253086656332016, "p95": 0.06693053245544434 }, { "label": "completed_group_6", "mean": 0.10493715852499008, "median": 0.09417732059955597, "p05": 0.04745320975780487, "p95": 0.19990575313568115 }, { "label": "completed_group_7", "mean": 0.31774085760116577, "median": 0.3171391487121582, "p05": 0.10361962765455246, "p95": 0.5363480448722839 }, { "label": "completed_group_8", "mean": 0.20717504620552063, "median": 0.19020375609397888, "p05": 0.03896218538284302, "p95": 0.4246461093425751 } ], "sources": 9 } ], "mode": "learned", "optimizer_state_after": "742c8682fdbe0ba8fa41dedba93757b7979af1e4c9d484b3e408e4a327c63f9d", "optimizer_state_before": "742c8682fdbe0ba8fa41dedba93757b7979af1e4c9d484b3e408e4a327c63f9d", "optimizer_state_unchanged": true, "positions": { "attention_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "e88f25858f6afc8c2818e9d252ea08bdd359890e32905783ed1bf3fce3e953f1", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.04804428666830063, 0.12356939166784286, 0.05923242121934891, 0.04508936405181885, 0.0799286738038063, 0.03290615975856781, 0.03170505166053772, 0.03961387276649475, 0.04508088901638985, 0.0312613770365715, 0.029994890093803406, 0.03058313950896263, 0.0497574545443058, 0.03759258985519409, 0.030690157786011696, 0.03000541403889656, 0.079553984105587, 0.0414140485227108, 0.04122982919216156, 0.05830265209078789, 0.03440643846988678, 0.033897366374731064, 0.02957344427704811, 0.03406442329287529, 0.025225287303328514, 0.01915411651134491, 0.014526215381920338, 0.011566666886210442, 0.0242468249052763, 0.013597902841866016, 0.020146220922470093, 0.02710210345685482 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.406048522767378e-05, "first_to_last_ratio": 1.2121174427344374, "imbalance_abs_log_ratio": 0.19236878289853168, "last_quartile_mean": 1.1599936385664478e-05, "mean": 1.385052358671146e-05, "normalized": [ 1.1035462446531923, 1.2453829183001823, 1.1319013131596403, 1.0360400584890102, 0.9887153549881315, 0.9324565992202007, 0.881809246461942, 0.8014211622598989, 0.7197327796271318, 0.6642367228932815, 0.63144486698815, 0.6054650987106983, 0.6612979512556375, 0.6318736594713652, 0.6080689788224007, 0.5826508433334544, 0.71610531341614, 0.685873407734308, 0.733241450138544, 0.7670951091759252, 2.8592552563843903, 2.5014923465112644, 2.101621489466362, 1.7092008525685705, 1.57037557605485, 1.4158002854110978, 1.11083466622777, 0.8417375710978994, 0.5124454971351892, 0.45441040448215947, 0.4041964735899746, 0.3902705019712373 ], "peak_layer": 21, "peak_normalized": 2.8592552563843903, "population_cv": 0.5792971940134687, "rest_mean": 1.0905006850710683e-05, "spike_contrast": 2.7286836559095273, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.9756313961115665e-05, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 1.5284693290595897e-05, 1.724920548440423e-05, 1.5677425835747272e-05, 1.4349697266879957e-05, 1.369422534480691e-05, 1.2915012121084146e-05, 1.2213519767101388e-05, 1.1100102710770443e-05, 9.96867584035499e-06, 9.20002639759332e-06, 8.745842023927253e-06, 8.38600863062311e-06, 9.159322871710174e-06, 8.75178102432983e-06, 8.422073733527213e-06, 8.070019248407334e-06, 9.91843353403965e-06, 9.499705811322201e-06, 1.0155777999898419e-05, 1.0624668902892154e-05, 3.960218236898072e-05, 3.464697874733247e-05, 2.910855801019352e-05, 2.3673326722928323e-05, 2.1750523956143297e-05, 1.9609575247159228e-05, 1.538564174552448e-05, 1.165850608231267e-05, 7.0976384449750185e-06, 6.293822025327245e-06, 5.598332791123539e-06, 5.405450792750344e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 5.784132736152969e-05, "first_to_last_ratio": 1.247704596204738, "imbalance_abs_log_ratio": 0.22130554016992907, "last_quartile_mean": 4.635819050236023e-05, "mean": 5.578696777774894e-05, "normalized": [ 1.1172279996665548, 1.2820128122064964, 1.1617993468115064, 1.0568066843619655, 1.0159130322231686, 0.9528514141416237, 0.8952059773271221, 0.8127833779089165, 0.726905325923399, 0.6687158255328219, 0.6353938715547037, 0.6079598423303039, 0.663249561462659, 0.632156813919224, 0.6072625305936415, 0.5816757612181556, 0.7067342161190907, 0.6745011066249388, 0.7095591340205663, 0.7592836625042008, 2.8322187742699385, 2.4767130392204146, 2.0818487421172924, 1.6933329620054323, 1.5551938962366243, 1.4028337924372432, 1.095017319874792, 0.8328031307001514, 0.5108569494896386, 0.4540995368567512, 0.4053097243347225, 0.3917738360059398 ], "peak_layer": 21, "peak_normalized": 2.8322187742699385, "population_cv": 0.5731363623317645, "rest_mean": 4.413512107459138e-05, "spike_contrast": 2.6896253388356377, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00011870693997479975, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 6.2326762417797e-05, 7.151960744522512e-05, 6.481326272478327e-05, 5.8956040447810665e-05, 5.667470759362914e-05, 5.315669113770127e-05, 4.994082701159641e-05, 4.534272011369467e-05, 4.055184399476275e-05, 3.730562821147032e-05, 3.5446697438601404e-05, 3.391623613424599e-05, 3.700068191392347e-05, 3.526611180859618e-05, 3.387733522686176e-05, 3.2449926948174834e-05, 3.942655894206837e-05, 3.762837150134146e-05, 3.9584152546012774e-05, 4.235813321429305e-05, 0.00015800089749973267, 0.00013816831051371992, 0.00011614002869464457, 9.446591138839722e-05, 8.675955177750438e-05, 7.825984357623383e-05, 6.108769593993202e-05, 4.645956141757779e-05, 2.8499160180217586e-05, 2.5332836230518296e-05, 2.2611000531469472e-05, 2.1855874365428463e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.051294267643243074, "first_to_last_ratio": 1.247704590273494, "imbalance_abs_log_ratio": 0.22130553541620454, "last_quartile_mean": 0.04111090721562505, "mean": 0.04947244143113494, "normalized": [ 1.1172280253192406, 1.2820128195205216, 1.1617993344751307, 1.056806678091042, 1.0159130361962296, 0.9528513588115517, 0.895205957602592, 0.8127833650766507, 0.726905316511733, 0.6687157952448587, 0.6353938256699311, 0.6079598134280393, 0.6632495949619723, 0.6321568158374328, 0.6072625701847152, 0.5816757499080466, 0.7067342453085952, 0.6745011182519967, 0.7095591365309404, 0.759283697871088, 2.8322188463523386, 2.476713119523555, 2.0818486723693663, 1.6933329451602226, 1.555193919529888, 1.4028338274869276, 1.095017294552204, 0.832803082651722, 0.5108569355338682, 0.4540995509098777, 0.40530972999858744, 0.391773821129134 ], "peak_layer": 21, "peak_normalized": 2.8322188463523386, "population_cv": 0.5731363736273818, "rest_mean": 0.039139466981093086, "spike_contrast": 2.6896253725737616, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.10527050346136094, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 0.055271998047828674, 0.06342430412769318, 0.05747704952955246, 0.05228280648589134, 0.05025969818234444, 0.047139883041381836, 0.0442880243062973, 0.04021037742495537, 0.035961780697107315, 0.033083003014326096, 0.03143448382616043, 0.0300772562623024, 0.03281257674098015, 0.03127434104681015, 0.03004276193678379, 0.028776919469237328, 0.03496386855840683, 0.033369217067956924, 0.03510362282395363, 0.037563618272542953, 0.14011678099632263, 0.1225290447473526, 0.10299413651227951, 0.08377331495285034, 0.07693924009799957, 0.06940161436796188, 0.05417317897081375, 0.041200801730155945, 0.025273339822888374, 0.022465413436293602, 0.020051661878824234, 0.01938200742006302 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 2.2204174854323355e-06, "first_to_last_ratio": 0.735976396978067, "imbalance_abs_log_ratio": 0.306557230090771, "last_quartile_mean": 3.0169683356007226e-06, "mean": 2.8222587751258743e-06, "normalized": [ 1.1437692142315412, 0.8934875077918236, 0.843634317775851, 0.8332025934148901, 0.6691897239669076, 0.649206320818976, 0.6345003713598615, 0.6270252408101402, 0.548073507234572, 0.5298155928110359, 0.5235851424549667, 0.5213985029212593, 0.5319259781097652, 0.5365763585413705, 0.538492744687615, 0.5418387466813859, 0.5975925562684307, 0.6133961155504078, 0.6328406634942718, 0.6938494895757952, 2.792643580637443, 2.792207888188673, 2.5604162828657895, 2.199406095874773, 1.9619910931916569, 1.8148930082375383, 1.4357221480925244, 1.0849940773120685, 0.6411781557251384, 0.5785168725769138, 0.5245362458580646, 0.5100938629385489 ], "peak_layer": 21, "peak_normalized": 2.792643580637443, "population_cv": 0.7010168394314351, "rest_mean": 2.0585069512014378e-06, "spike_contrast": 3.3745422235584535, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.946518624317832e-06, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 3.2280127015837934e-06, 2.5216529593308223e-06, 2.3809543563402258e-06, 2.3515133307228098e-06, 1.8886265706896666e-06, 1.8322282357985387e-06, 1.7907242408909951e-06, 1.7696274881018326e-06, 1.5468052652067854e-06, 1.495276706009463e-06, 1.4776927628190606e-06, 1.4715215002070181e-06, 1.5012327594376984e-06, 1.5143573364184704e-06, 1.5197658740362385e-06, 1.5292091575247468e-06, 1.6865608358784812e-06, 1.7311625697402633e-06, 1.7860401158031891e-06, 1.958222810571897e-06, 7.881562851252966e-06, 7.880333214416169e-06, 7.2261573222931474e-06, 6.207293154147919e-06, 5.5372465794789605e-06, 5.122097718412988e-06, 4.051979431096697e-06, 3.062134055653587e-06, 1.809570676414296e-06, 1.6327243201885722e-06, 1.4803770227445057e-06, 1.4396168808161747e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 4.569237398754922e-05, "first_to_last_ratio": 1.4673112290466532, "imbalance_abs_log_ratio": 0.38343163006552816, "last_quartile_mean": 3.114020603334211e-05, "mean": 4.183441066629712e-05, "normalized": [ 1.1974296567066158, 1.344289107904746, 1.230408462214659, 1.1234591616583915, 1.0623659972849115, 0.9983070144562474, 0.9335099902486312, 0.8479894563327729, 0.7756304429380606, 0.7156796904410139, 0.6795815361308393, 0.6491943107046658, 0.7109629912124332, 0.6750948063174392, 0.6474411689349395, 0.6200717213988419, 0.7662657439847727, 0.7307653188389935, 0.7545343036423542, 0.7927280102241547, 2.8287575533506253, 2.4215604626110183, 1.9807235727824308, 1.5583037155212895, 1.4117479335225582, 1.2389201568217543, 0.9856291732269274, 0.776844076198889, 0.43766749474230543, 0.39060885635754505, 0.36143095960303334, 0.352097153686139 ], "peak_layer": 21, "peak_normalized": 2.8287575533506253, "population_cv": 0.551495636352156, "rest_mean": 3.3775719169137085e-05, "spike_contrast": 2.5270030320761303, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 8.535134475096129e-05, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 5.009376400266774e-05, 5.623754259431735e-05, 5.1473412895575166e-05, 4.699925193563104e-05, 4.444345540832728e-05, 4.176358561380766e-05, 3.905284029315226e-05, 3.547513915691525e-05, 3.244804247515276e-05, 2.994003807543777e-05, 2.842989306373056e-05, 2.7158661396242678e-05, 2.9742717742919922e-05, 2.8242193366168067e-05, 2.708531974349171e-05, 2.5940335035556927e-05, 3.205627581337467e-05, 3.0571136448998004e-05, 3.156549792038277e-05, 3.316330912639387e-05, 0.00011833940516225994, 0.00010130455484613776, 8.286240336019546e-05, 6.519071757793427e-05, 5.905964280827902e-05, 5.1829494623234496e-05, 4.1233215597458184e-05, 3.249881410738453e-05, 1.830956171033904e-05, 1.63408913067542e-05, 1.512025119154714e-05, 1.4729776921740267e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 3.5050525639235275e-05, "first_to_last_ratio": 1.5890428234254472, "imbalance_abs_log_ratio": 0.46313183711305195, "last_quartile_mean": 2.2057634396333015e-05, "mean": 3.131621542706853e-05, "normalized": [ 1.247642034455602, 1.3675088708957686, 1.2617968995203173, 1.1676693179472686, 1.0797799047255963, 1.0154085918080462, 0.9450439951530737, 0.8691124556208352, 0.8151155724977966, 0.7531778294737046, 0.7147915273060139, 0.6827252927557514, 0.7417922051240886, 0.7100839458764344, 0.679513505581043, 0.6487030016923234, 0.8192115232383504, 0.7896113859496233, 0.7975027577835305, 0.8129328118460493, 2.8268246118214386, 2.3622917424367986, 1.850934565013104, 1.4060108194738914, 1.2902617154939733, 1.123282471468924, 0.9231284485726192, 0.7885511099194735, 0.4159101173561302, 0.376982553975174, 0.35968054987980413, 0.3570178653374513 ], "peak_layer": 21, "peak_normalized": 2.8268246118214386, "population_cv": 0.5277183722759345, "rest_mean": 2.5822744107615478e-05, "spike_contrast": 2.3615213122965844, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.0980960552114996e-05, "top_five_layers": [ 21, 22, 23, 24, 2 ], "top_five_spike_overlap": 4 }, "values": [ 3.907142672687769e-05, 4.282520239939913e-05, 3.951470353058539e-05, 3.656698390841484e-05, 3.3814620110206306e-05, 3.179875420755707e-05, 2.959520134027116e-05, 2.721731289057061e-05, 2.5526334866299294e-05, 2.3586679162690416e-05, 2.238456545455847e-05, 2.138037234544754e-05, 2.3230124497786164e-05, 2.223714182036929e-05, 2.1279791326378472e-05, 2.03149229491828e-05, 2.5654604542069137e-05, 2.4727640266064554e-05, 2.4974768166430295e-05, 2.5457979063503444e-05, 8.852544851833954e-05, 7.397803710773587e-05, 5.796426557935774e-05, 4.403093771543354e-05, 4.040611383970827e-05, 3.517695586197078e-05, 2.8908889362355694e-05, 2.469443643349223e-05, 1.3024730833421927e-05, 1.1805666872533038e-05, 1.1263833584962413e-05, 1.1180448382219765e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00011513851586641977, "first_to_last_ratio": 1.1690722131114868, "imbalance_abs_log_ratio": 0.15621045398792427, "last_quartile_mean": 9.848708623394486e-05, "mean": 0.00011408905686494109, "normalized": [ 1.0878105091943224, 1.2333698943486997, 1.1325669787296346, 1.0350609428916697, 0.9847532971789725, 0.9327565235282874, 0.875161061795294, 0.7921095504711437, 0.7054552974920509, 0.6480774339061697, 0.6173745394266138, 0.5959457136906282, 0.6486626275163402, 0.6167276122272274, 0.5953045899588231, 0.5685580660104519, 0.6801746911900812, 0.6508152675683995, 0.6710893308044551, 0.7221277670118063, 2.8258795357154587, 2.493070695082837, 2.1580463328702972, 1.8231222239658644, 1.644278740520026, 1.5034127662796515, 1.171932855948766, 0.8973146194608761, 0.48937000596181734, 0.4306716621780954, 0.38968515069330856, 0.3793137163819305 ], "peak_layer": 21, "peak_normalized": 2.8258795357154587, "population_cv": 0.5949686549165712, "rest_mean": 8.89708825087399e-05, "spike_contrast": 2.806841871708938, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0002497271983884275, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 0.00012410727504175156, 0.00014071400801185519, 0.0001292134984396398, 0.00011808912677224725, 0.00011234957491979003, 0.00010641731205396354, 9.98463001451455e-05, 9.037103154696524e-05, 8.048472955124453e-05, 7.393854320980608e-05, 7.043567893560976e-05, 6.799088441766798e-05, 7.400530739687383e-05, 7.036187162157148e-05, 6.79177392157726e-05, 6.486625352408737e-05, 7.760048902127892e-05, 7.425090007018298e-05, 7.656394882360473e-05, 8.238687587436289e-05, 0.0003224019310437143, 0.00028443208429962397, 0.0002462094707880169, 0.00020799829508177936, 0.00018759421072900295, 0.00017152294458355755, 0.00013370471424423158, 0.00010237377864541486, 5.583176243817434e-05, 4.913492375635542e-05, 4.4458811316872016e-05, 4.327554415795021e-05 ] } } }, "mlp_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "dba9c602cebba42dc99cd260f60389d5962764f3da1c351817656a50dba8a4c1", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.07808538526296616, 0.03881331905722618, 0.01480295043438673, 0.02500341460108757, 0.015180142596364021, 0.013721407391130924, 0.016902483999729156, 0.03325798362493515, 0.026172807440161705, 0.027887089177966118, 0.03240044787526131, 0.03682815283536911, 0.029301028698682785, 0.0338764488697052, 0.0393521711230278, 0.046694882214069366, 0.029503758996725082, 0.03615153953433037, 0.030687781050801277, 0.05279085040092468, 0.024021649733185768, 0.03336174041032791, 0.04255227372050285, 0.0520954355597496, 0.03984410688281059, 0.04249191656708717, 0.05309459939599037, 0.06231515854597092, 0.06388300657272339, 0.0749494656920433, 0.10156872868537903, 0.11044161766767502 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.387307077038713e-05, "first_to_last_ratio": 1.3885610977109948, "imbalance_abs_log_ratio": 0.32826802946929257, "last_quartile_mean": 9.990968919737497e-06, "mean": 1.2854331032485788e-05, "normalized": [ 1.1519258940103256, 1.3312041641813053, 1.2167309231197612, 1.1117514471382886, 1.0587713135165344, 0.9928224052891267, 0.9274023828418168, 0.8434126323121538, 0.7336839389102785, 0.6915061646833663, 0.6596107083150127, 0.6376340360074726, 0.6952642607649582, 0.6629197998266377, 0.6345516401532968, 0.6122226131721931, 0.7648990252904823, 0.729764382313612, 0.7866676809835089, 0.8716429632898921, 2.8889885153646824, 2.3385372141657035, 1.899664892928511, 1.5404582652892223, 1.5315814746576397, 1.1978299957950485, 0.9082536044284508, 0.816087598131252, 0.48962660632970234, 0.43639989519040423, 0.42103372910248354, 0.4171498324968764 ], "peak_layer": 21, "peak_normalized": 2.8889885153646824, "population_cv": 0.5419279223583527, "rest_mean": 1.0379048506837097e-05, "spike_contrast": 2.5263256698064365, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.6220856670988724e-05, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 1.4807236766500864e-05, 1.7111738998210058e-05, 1.5640262063243426e-05, 1.4290821127360687e-05, 1.3609796951641329e-05, 1.2762067854055203e-05, 1.1921137229364831e-05, 1.0841505172720645e-05, 9.4310162239708e-06, 8.888849151844624e-06, 8.478854397253599e-06, 8.196358976420015e-06, 8.937156962929294e-06, 8.521390554960817e-06, 8.156736839737277e-06, 7.869712135288864e-06, 9.832265277509578e-06, 9.380632945976686e-06, 1.0112086783919949e-05, 1.120438719226513e-05, 3.7136014725547284e-05, 3.0060331482673064e-05, 2.441892138449475e-05, 1.9801560483756475e-05, 1.9687455278472044e-05, 1.5397303286590613e-05, 1.1674992492771707e-05, 1.0490260137885343e-05, 6.293822480074596e-06, 5.609628715319559e-06, 5.412106929725269e-06, 5.3621820370608475e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 5.7066908539127326e-05, "first_to_last_ratio": 1.4278402189707298, "imbalance_abs_log_ratio": 0.3561629661799943, "last_quartile_mean": 3.99672931052919e-05, "mean": 5.182235634038079e-05, "normalized": [ 1.1683197832075227, 1.369358043457492, 1.2477885257362629, 1.1320828936276526, 1.0861971638477104, 1.0136921452638399, 0.9398491667147963, 0.8523323061186183, 0.7376381293423032, 0.6943242114792568, 0.6618878610984286, 0.637951161315391, 0.6961169332186252, 0.6619150990682322, 0.6320240246440271, 0.6099217459665006, 0.753292310075495, 0.715605419809301, 0.7584607148457314, 0.8619138236046655, 2.8636215539276253, 2.323250279031787, 1.8821006100447468, 1.5304640359562964, 1.5177704400182797, 1.1794085853576253, 0.8974069340864187, 0.8090474514501561, 0.4887596394765995, 0.4369840848318672, 0.4222231761264232, 0.4182917472503225 ], "peak_layer": 21, "peak_normalized": 2.8636215539276253, "population_cv": 0.5384897216371255, "rest_mean": 4.2000662991389014e-05, "spike_contrast": 2.4966153616773794, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00010485950042493641, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 6.0545084124896675e-05, 7.09633604856208e-05, 6.466334161814302e-05, 5.866720312042162e-05, 5.6289296480827034e-05, 5.2531915571307763e-05, 4.870519842370413e-05, 4.416986848809756e-05, 3.822614598902874e-05, 3.598151670303196e-05, 3.430058859521523e-05, 3.306013240944594e-05, 3.607441976782866e-05, 3.430200013099238e-05, 3.275297422078438e-05, 3.1607582059223205e-05, 3.9037382521200925e-05, 3.708435906446539e-05, 3.930522143491544e-05, 4.466640530154109e-05, 0.00014839961659163237, 0.00012039630382787436, 9.753488848218694e-05, 7.931225263746455e-05, 7.865444058552384e-05, 6.111973198130727e-05, 4.650574192055501e-05, 4.192674532532692e-05, 2.5328676201752387e-05, 2.264554495923221e-05, 2.188059988839086e-05, 2.1676863980246708e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.05060750572010875, "first_to_last_ratio": 1.4278402181986947, "imbalance_abs_log_ratio": 0.356162965639293, "last_quartile_mean": 0.03544339560903609, "mean": 0.045956584566738456, "normalized": [ 1.1683197838664416, 1.3693581481609842, 1.2477884983122534, 1.1320828879264566, 1.086197129718159, 1.0136921293230559, 0.9398491347700728, 0.852332271168493, 0.7376380598562373, 0.6943242075392044, 0.6618878602537814, 0.6379511317995364, 0.6961168732850856, 0.66191505624554, 0.6320240673482492, 0.6099217105646539, 0.7532923321079824, 0.7156054155999519, 0.7584607053972269, 0.8619137716807654, 2.8636216906024767, 2.3232503851911614, 1.8821005956556751, 1.530464123018414, 1.5177704445282252, 1.1794086151290946, 0.8974069322276481, 0.8090474387174796, 0.48875963581745097, 0.43698406072332996, 0.42222316261516873, 0.41829174084974413 ], "peak_layer": 21, "peak_normalized": 2.8636216906024767, "population_cv": 0.5384897545714166, "rest_mean": 0.03724660800287017, "spike_contrast": 2.496615477158658, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0929904580116272, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 0.053691986948251724, 0.06293102353811264, 0.05734409764409065, 0.0520266629755497, 0.04991791024804115, 0.04658582806587219, 0.04319225624203682, 0.0391702800989151, 0.033899325877428055, 0.03190876916050911, 0.03041810542345047, 0.029318055137991905, 0.031991153955459595, 0.030419355258345604, 0.029045667499303818, 0.028029918670654297, 0.034618742763996124, 0.03288678079843521, 0.03485626354813576, 0.0396106131374836, 0.13160227239131927, 0.1067686527967453, 0.08649491518735886, 0.07033490389585495, 0.0697515457868576, 0.05420159175992012, 0.041241757571697235, 0.037181057035923004, 0.022461723536252975, 0.020082294940948486, 0.019403934478759766, 0.019223259761929512 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 2.187219166671639e-06, "first_to_last_ratio": 0.8437183001333121, "imbalance_abs_log_ratio": 0.16993660771354646, "last_quartile_mean": 2.5923571485009234e-06, "mean": 2.5683367930184886e-06, "normalized": [ 1.1919543817128102, 0.9792836739870964, 0.924042037649071, 0.9164810837361373, 0.7326981846944463, 0.7055637920842305, 0.6863211882787902, 0.6765290243634249, 0.5799994679651287, 0.5635357621627565, 0.5597470962238182, 0.5639314892075872, 0.5739189935534712, 0.5756380159008986, 0.5748207554502599, 0.5781987328520684, 0.6522355446992087, 0.6638052489880939, 0.6844497180630856, 0.7655738065181559, 2.843883820310657, 2.5944552048335017, 2.2989551083840256, 2.0391579131550635, 1.9644428765002167, 1.5269231604355886, 1.179167971687766, 1.0934476508277742, 0.6272800411331483, 0.5710254497958045, 0.5563528736513774, 0.556179931194537 ], "peak_layer": 21, "peak_normalized": 2.843883820310657, "population_cv": 0.6360421157910927, "rest_mean": 1.9271187023116874e-06, "spike_contrast": 3.129498185867219, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.030914482835214e-06, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 3.0613402941526147e-06, 2.5151302907033823e-06, 2.3732511635898845e-06, 2.35383208746498e-06, 1.8818157059286023e-06, 1.8121254470315762e-06, 1.7627039596845862e-06, 1.7375543848174857e-06, 1.4896339735059883e-06, 1.4473496321443236e-06, 1.4376190620168927e-06, 1.4483659924735548e-06, 1.4740172673555207e-06, 1.4784322956984397e-06, 1.4763332956135855e-06, 1.4850090792606352e-06, 1.675160547165433e-06, 1.7048754443749203e-06, 1.757897393872554e-06, 1.9662513750517974e-06, 7.304051450773841e-06, 6.663434760412201e-06, 5.9044909903605e-06, 5.2372442951309495e-06, 5.045350917498581e-06, 3.921652933058795e-06, 3.028500486834673e-06, 2.808341832860606e-06, 1.6110664091684157e-06, 1.4665856724604964e-06, 1.4289015553003992e-06, 1.4284573808254208e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 4.514474267125479e-05, "first_to_last_ratio": 1.655519660200222, "imbalance_abs_log_ratio": 0.5041149536218862, "last_quartile_mean": 2.726922775764251e-05, "mean": 3.928292341015549e-05, "normalized": [ 1.2428531532199, 1.4224719349524337, 1.3075943550391356, 1.1908940314839356, 1.1240062299860272, 1.0512639226040563, 0.9724687394121546, 0.8822119975062471, 0.7835780621436017, 0.7378068411250925, 0.7017901972736155, 0.675554292822624, 0.7400896696370771, 0.7012250467261758, 0.6687293301443596, 0.6449026664130145, 0.8086596332888067, 0.7678950765711053, 0.7972794784799546, 0.884192918474659, 2.849023696568842, 2.282552164711599, 1.7952274549858118, 1.4143282104887471, 1.3505434814026465, 1.05691913246549, 0.8328089954359738, 0.7560703953177289, 0.41728160395883906, 0.38644818475535425, 0.376537560284359, 0.3767915423206315 ], "peak_layer": 21, "peak_normalized": 2.849023696568842, "population_cv": 0.5250932744673414, "rest_mean": 3.245689711345929e-05, "spike_contrast": 2.34598720716096, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 7.614346541231498e-05, "top_five_layers": [ 21, 22, 23, 2, 24 ], "top_five_spike_overlap": 4 }, "values": [ 4.8822905228007585e-05, 5.587885607383214e-05, 5.136612890055403e-05, 4.6781799028394744e-05, 4.415425064507872e-05, 4.129672015551478e-05, 3.820141500909813e-05, 3.4655866329558194e-05, 3.0781237001065165e-05, 2.898320963140577e-05, 2.7568370569497347e-05, 2.6537747544352897e-05, 2.907288580900058e-05, 2.754616980382707e-05, 2.6269643058185466e-05, 2.5333662051707506e-05, 3.176651443936862e-05, 3.016516347997822e-05, 3.131946868961677e-05, 3.4733682696241885e-05, 0.0001119179796660319, 8.966532186605036e-05, 7.0521782618016e-05, 5.555894676945172e-05, 5.305329614202492e-05, 4.151887333136983e-05, 3.271517198299989e-05, 2.9700655431952327e-05, 1.639204128878191e-05, 1.51808144437382e-05, 1.479149614169728e-05, 1.4801473298575729e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 3.469924399723823e-05, "first_to_last_ratio": 1.7510149859622042, "imbalance_abs_log_ratio": 0.5601956117834839, "last_quartile_mean": 1.981664593131427e-05, "mean": 2.9772959720730796e-05, "normalized": [ 1.2826990789388135, 1.4313521308000512, 1.325603166494873, 1.2222410753351367, 1.1310087328328617, 1.056511628684211, 0.9758416336275024, 0.898436085327916, 0.8214143991142518, 0.7703245121438669, 0.7298179894195511, 0.7040392340287683, 0.7681958890441348, 0.7313866125384519, 0.6969281018261846, 0.6697043803055042, 0.8568340618774071, 0.8208415080145076, 0.8328878615188123, 0.8883662275011015, 2.8312616617599446, 2.239848895619399, 1.6996416088520014, 1.290076953440798, 1.215112714243769, 0.9719493099553681, 0.8309033623504332, 0.7782938507320865, 0.3989932047584423, 0.3786265198784562, 0.3759890641537673, 0.37486854488162713 ], "peak_layer": 21, "peak_normalized": 2.8312616617599446, "population_cv": 0.5072717351311633, "rest_mean": 2.505786920260612e-05, "spike_contrast": 2.2042755539987984, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.523444851860404e-05, "top_five_layers": [ 21, 22, 23, 2, 3 ], "top_five_spike_overlap": 3 }, "values": [ 3.8189748011063784e-05, 4.261558933649212e-05, 3.9467129681725055e-05, 3.638973430497572e-05, 3.367347744642757e-05, 3.14554781652987e-05, 2.905369365180377e-05, 2.67491013801191e-05, 2.445593781885691e-05, 2.293484067195095e-05, 2.172884160245303e-05, 2.096133175655268e-05, 2.287146526214201e-05, 2.177554415538907e-05, 2.0749612303916365e-05, 1.9939081539632753e-05, 2.55104860116262e-05, 2.443888115521986e-05, 2.4797536752885208e-05, 2.6449291908647865e-05, 8.429503941442817e-05, 6.668693094979972e-05, 5.0603361160028726e-05, 3.840940917143598e-05, 3.61775018973276e-05, 2.8937807655893266e-05, 2.473845233907923e-05, 2.3172111468738876e-05, 1.1879208614118397e-05, 1.1272832125541754e-05, 1.1194307262485381e-05, 1.1160946087329648e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00011354441994626541, "first_to_last_ratio": 1.3378984746048481, "imbalance_abs_log_ratio": 0.2911000803554069, "last_quartile_mean": 8.486774004268227e-05, "mean": 0.00010590371198304638, "normalized": [ 1.1416862116188895, 1.318287506150763, 1.2188213004978723, 1.1129317281051134, 1.0503469451456144, 0.9936779473327889, 0.9152461482494265, 0.8261837163725879, 0.7163781808438098, 0.6746444347634984, 0.649936107552919, 0.6258996320477875, 0.6816391399615429, 0.6448073899118816, 0.6202888218188445, 0.5944257945762098, 0.72764205396545, 0.6903920368963828, 0.7174952313478369, 0.8067894001918988, 2.8841338501609886, 2.3452358192551364, 1.972412467214386, 1.6597621298267584, 1.6194268438614279, 1.2623302767875173, 0.9611591980513982, 0.8707097795471348, 0.4633879176416832, 0.4198531088291576, 0.4079853661334264, 0.40608351533986714 ], "peak_layer": 21, "peak_normalized": 2.8841338501609886, "population_cv": 0.5599416174982532, "rest_mean": 8.440537176620974e-05, "spike_contrast": 2.630102142892712, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00022199474915396422, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 0.00012090880773030221, 0.00013961154036223888, 0.0001290776999667287, 0.00011786360119003803, 0.00011123564036097378, 0.0001052341831382364, 9.692796447779983e-05, 8.749592234380543e-05, 7.586710853502154e-05, 7.144734991015866e-05, 6.883064634166658e-05, 6.62850943626836e-05, 7.218811515485868e-05, 6.828749610576779e-05, 6.569088873220608e-05, 6.295189814409241e-05, 7.70599945099093e-05, 7.311507943086326e-05, 7.598540832987055e-05, 8.544199226889759e-05, 0.000305440480588004, 0.00024836917873471975, 0.00020888580183964223, 0.00017577497055754066, 0.00017150331404991448, 0.00013368546206038445, 0.0001017903268802911, 9.221139771398157e-05, 4.9074500566348433e-05, 4.4464002712629735e-05, 4.320716470829211e-05, 4.3005751649616286e-05 ] } } }, "post_attention_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "461f23fe7003cf2bffe4fbf4022ddcdfd511968996b161f3081b3d083dec0aab", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.04804428666830063, 0.18500657379627228, 0.22299833595752716, 0.2331935465335846, 0.0799286738038063, 0.09402555972337723, 0.10843002796173096, 0.12457633018493652, 0.04508088901638985, 0.0760042667388916, 0.10498681664466858, 0.13159531354904175, 0.0497574545443058, 0.07735268771648407, 0.10224150121212006, 0.13362281024456024, 0.079553984105587, 0.10536550730466843, 0.14260481297969818, 0.19080305099487305, 0.03440643846988678, 0.060867179185152054, 0.08364810794591904, 0.11119203269481659, 0.025225287303328514, 0.05530283972620964, 0.08037404716014862, 0.11502843350172043, 0.0242468249052763, 0.07164724916219711, 0.1371813863515854, 0.23144805431365967 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.406055810093676e-05, "first_to_last_ratio": 1.2121225310488972, "imbalance_abs_log_ratio": 0.19237298076220427, "last_quartile_mean": 1.1599947811191669e-05, "mean": 1.3850583115981863e-05, "normalized": [ 1.1035219335667126, 1.245398841062786, 1.1319237648162956, 1.0360361966114253, 0.9887036854141024, 0.9324573194178568, 0.8818082800629979, 0.8014300627506455, 0.7197404552580381, 0.6642341963531577, 0.63144215306254, 0.605460329509273, 0.6612975386173722, 0.6318610283296698, 0.6080617688348535, 0.5826465005104303, 0.7161183891450303, 0.6858808349031477, 0.733242501237421, 0.767094241825568, 2.859288144745241, 2.5015026079052487, 2.101601950422847, 1.7091984970083665, 1.5703665940407991, 1.4157861892600545, 1.1108502479674358, 0.8417170775086074, 0.5124528160490821, 0.4544107825414821, 0.40419536018403524, 0.39026971107747666 ], "peak_layer": 21, "peak_normalized": 2.859288144745241, "population_cv": 0.5793000123245652, "rest_mean": 1.0905047424279877e-05, "spike_contrast": 2.7286883489310063, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.975647585117258e-05, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 1.5284422261174768e-05, 1.7249500160687603e-05, 1.567780418554321e-05, 1.4349705452332273e-05, 1.369412257190561e-05, 1.2915077604702674e-05, 1.2213558875373565e-05, 1.1100273695774376e-05, 9.968824997486081e-06, 9.200030945066828e-06, 8.745842023927253e-06, 8.385978617297951e-06, 9.15935652301414e-06, 8.751643690629862e-06, 8.42201006889809e-06, 8.069993782555684e-06, 9.918657269736286e-06, 9.499849511485081e-06, 1.0155836207559332e-05, 1.062470255419612e-05, 3.960280810133554e-05, 3.4647269785637036e-05, 2.9108412491041236e-05, 2.3673395844525658e-05, 2.1750493033323437e-05, 1.960946428880561e-05, 1.538592368888203e-05, 1.1658272342174314e-05, 7.0977703217067756e-06, 6.293854312389158e-06, 5.598341431323206e-06, 5.405463070928818e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 5.784146014775615e-05, "first_to_last_ratio": 1.2477003862842722, "imbalance_abs_log_ratio": 0.22130216603186706, "last_quartile_mean": 4.635845334632904e-05, "mean": 5.5787147289265704e-05, "normalized": [ 1.1172174922096436, 1.2820167732139756, 1.1618030425233523, 1.0568058922472814, 1.015898285952758, 0.952837653331755, 0.8952147696466347, 0.8127990870482149, 0.7269196158944218, 0.6687128912021425, 0.6353920226237901, 0.6079547558759868, 0.6632448840034066, 0.632152432145331, 0.6072547727004206, 0.5816761067060003, 0.7067464190080565, 0.6745056530323595, 0.7095637632538165, 0.7592902185109378, 2.8322130517871535, 2.4767089823758397, 2.0818425648522476, 1.6933243830446072, 1.5552023255692202, 1.402829278407864, 1.095030881819868, 0.8327931471930783, 0.5108623485290222, 0.45409986898200716, 0.4053123002322658, 0.3917743360765406 ], "peak_layer": 21, "peak_normalized": 2.8322130517871535, "population_cv": 0.5731345570971216, "rest_mean": 4.41352963712739e-05, "spike_contrast": 2.6896192391648626, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00011870714224642143, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 6.232637679204345e-05, 7.15200585545972e-05, 6.481367745436728e-05, 5.895618596696295e-05, 5.667406730935909e-05, 5.315609450917691e-05, 4.994147820980288e-05, 4.534374238573946e-05, 4.055277167935856e-05, 3.7305584555724636e-05, 3.5446708352537826e-05, 3.391606151126325e-05, 3.7000540032749996e-05, 3.5265980841359124e-05, 3.387701144674793e-05, 3.2450050639454275e-05, 3.942736657336354e-05, 3.762874621315859e-05, 3.9584538171766326e-05, 4.2358635255368426e-05, 0.00015800108667463064, 0.00013816852879244834, 0.00011614005779847503, 9.446573676541448e-05, 8.676030120113865e-05, 7.825984357623383e-05, 6.108864909037948e-05, 4.6459153963951394e-05, 2.8499553081928752e-05, 2.533293627493549e-05, 2.2611216991208494e-05, 2.1855972590856254e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.051294384989887476, "first_to_last_ratio": 1.2477003995429623, "imbalance_abs_log_ratio": 0.2213021766583686, "last_quartile_mean": 0.04111113934777677, "mean": 0.04947259998880327, "normalized": [ 1.117217517042371, 1.282016692529182, 1.1618030656626175, 1.0568059265711638, 1.0158982593254349, 0.9528376876517187, 0.8952147600149789, 0.8127990580563501, 0.7269196281255159, 0.6687128990372552, 0.6353920151588079, 0.6079547776386781, 0.6632449090716109, 0.6321524554983756, 0.6072547505273581, 0.5816761070130537, 0.706746437867777, 0.674505658209711, 0.7095637900293472, 0.7592902251062212, 2.8322130824094, 2.4767089467624213, 2.081842602529718, 1.693324355494102, 1.5552023386093694, 1.4028291808614948, 1.0950308028824434, 0.8327931847449266, 0.5108623388148443, 0.45409990274101625, 0.40531230895107145, 0.3917743350616629 ], "peak_layer": 21, "peak_normalized": 2.8322130824094, "population_cv": 0.5731345524386317, "rest_mean": 0.03913962206354848, "spike_contrast": 2.689619246048363, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.10527068078517914, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 0.055271655321121216, 0.06342469900846481, 0.05747741833329201, 0.05228293687105179, 0.05025912821292877, 0.04713935777544975, 0.044288601726293564, 0.040211282670497894, 0.035962603986263275, 0.03308296576142311, 0.031434495002031326, 0.030077103525400162, 0.03281245008111, 0.0312742255628109, 0.03004247136414051, 0.028777029365301132, 0.034964583814144135, 0.03336954861879349, 0.03510396555066109, 0.03756406158208847, 0.14011694490909576, 0.12252923101186752, 0.1029941663146019, 0.0837731584906578, 0.0769399031996727, 0.06940160691738129, 0.054174020886421204, 0.04120044410228729, 0.02527368813753128, 0.022465502843260765, 0.020051853731274605, 0.019382094964385033 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 2.2204297493999547e-06, "first_to_last_ratio": 0.7359606295430359, "imbalance_abs_log_ratio": 0.3065786541527352, "last_quartile_mean": 3.0170496359005483e-06, "mean": 2.8223206918198684e-06, "normalized": [ 1.1437255119946472, 0.8934281083094181, 0.843667370045445, 0.833234424409991, 0.6691350034907739, 0.6491575975670105, 0.6344872974587108, 0.6270766601877794, 0.5480904457549984, 0.5298089242035497, 0.5236022556745512, 0.5213819889179987, 0.5319070176791533, 0.5365159271613865, 0.5384760973551012, 0.5417839198031603, 0.5976312076631317, 0.6133914803428778, 0.632878138794897, 0.6938486884695898, 2.7927450515890637, 2.792273921126163, 2.5604223062656137, 2.1993623563278324, 1.9620758229843447, 1.8148751057591102, 1.435693067783232, 1.0849680187180202, 0.6411889832806804, 0.5785351975720235, 0.5245373868045311, 0.5100947165052133 ], "peak_layer": 21, "peak_normalized": 2.7927450515890637, "population_cv": 0.7010322041721825, "rest_mean": 2.058529688568982e-06, "spike_contrast": 3.374637804812921, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.946792109374656e-06, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 3.227960178264766e-06, 2.5215406367351534e-06, 2.3810998754925095e-06, 2.3516547571489355e-06, 1.8885135659729713e-06, 1.832130919865449e-06, 1.7907266283145873e-06, 1.7698114334052661e-06, 1.5468870060431072e-06, 1.4952906894905027e-06, 1.477773480473843e-06, 1.4715071756654652e-06, 1.5012121821200708e-06, 1.5142200027185027e-06, 1.519752231615712e-06, 1.5290879673557356e-06, 1.6867069234649534e-06, 1.7311874671577243e-06, 1.7861850665212842e-06, 1.958263510459801e-06, 7.882022146077361e-06, 7.880692464823369e-06, 7.22633285477059e-06, 6.207305887073744e-06, 5.5376071941282135e-06, 5.1221595640527084e-06, 4.0519862523069605e-06, 3.0621276891906746e-06, 1.809640934880008e-06, 1.6328118590536178e-06, 1.4804127204115503e-06, 1.4396508731806534e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 4.56924158243055e-05, "first_to_last_ratio": 1.4673115279536288, "imbalance_abs_log_ratio": 0.38343183377619244, "last_quartile_mean": 3.1140228202275466e-05, "mean": 4.183447202876778e-05, "normalized": [ 1.1974206825409346, 1.3442935712461421, 1.2304058748098903, 1.1234622096860003, 1.0623591343774659, 0.9983032021931795, 0.9335117515893174, 0.8479976043257199, 0.7756355664609215, 0.7156793363778972, 0.6795817133031917, 0.6491928801846335, 0.7109607744004338, 0.6750928595197824, 0.6474397844670364, 0.6200735511625727, 0.7662731422367131, 0.7307687689442663, 0.7545377188834692, 0.7927345870100259, 2.8287570565248865, 2.421556562840323, 1.9807276243744367, 1.558292733686825, 1.4117490803489046, 1.238912861023533, 0.9856350322605589, 0.7768378929765343, 0.43766980945827866, 0.39060663115145766, 0.36143264697135674, 0.3520973546633113 ], "peak_layer": 21, "peak_normalized": 2.8287570565248865, "population_cv": 0.551494708887264, "rest_mean": 3.377578448433067e-05, "spike_contrast": 2.5269993301954115, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 8.535138476872817e-05, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 5.009346205042675e-05, 5.623781180474907e-05, 5.14733801537659e-05, 4.699944838648662e-05, 4.444323349162005e-05, 4.176348738837987e-05, 3.905297126038931e-05, 3.547553205862641e-05, 3.244830440962687e-05, 2.9940067179268226e-05, 2.8429942176444456e-05, 2.715864138735924e-05, 2.9742668630206026e-05, 2.824215334840119e-05, 2.7085301553597674e-05, 2.594044963188935e-05, 3.205663233529776e-05, 3.057132562389597e-05, 3.156568709528074e-05, 3.31636329065077e-05, 0.00011833955795736983, 0.00010130454029422253, 8.286269439850003e-05, 6.519035378005356e-05, 5.9059777413494885e-05, 5.182926543056965e-05, 4.123352118767798e-05, 3.249860310461372e-05, 1.830968540161848e-05, 1.6340822185156867e-05, 1.5120343960006721e-05, 1.4729806935065426e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 3.505123777358676e-05, "first_to_last_ratio": 1.5889493494930538, "imbalance_abs_log_ratio": 0.4630730113341737, "last_quartile_mean": 2.2059380171413068e-05, "mean": 3.131689976498819e-05, "normalized": [ 1.2479593211749789, 1.367487352044708, 1.261680923885286, 1.1674855830446973, 1.0794756507539984, 1.0156514953231304, 0.945160188314607, 0.8690478102601445, 0.8149681766862981, 0.7530914967772463, 0.7148635553828698, 0.6827279730652124, 0.7417389963838543, 0.7102014979770842, 0.6794125192711595, 0.6486601330546842, 0.8192809210261347, 0.7895373258521271, 0.7975483511256357, 0.812938977938045, 2.826829287271584, 2.3623544295272914, 1.8510256189591003, 1.4057347513501408, 1.2902085448265628, 1.123507683724877, 0.9230685473509727, 0.7887132397564032, 0.4159196155359273, 0.3769758263062851, 0.35976736593467973, 0.35697684011427405 ], "peak_layer": 21, "peak_normalized": 2.826829287271584, "population_cv": 0.5277213122295441, "rest_mean": 2.5823506533655477e-05, "spike_contrast": 2.3614617610009216, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.0981223214184865e-05, "top_five_layers": [ 21, 22, 23, 24, 2 ], "top_five_spike_overlap": 4 }, "values": [ 3.908221697201952e-05, 4.282546433387324e-05, 3.9511935028713197e-05, 3.656202898127958e-05, 3.380583075340837e-05, 3.180705607519485e-05, 2.9599486879305914e-05, 2.7215883164899424e-05, 2.5522276700939983e-05, 2.358449091843795e-05, 2.238731030956842e-05, 2.1380923499236815e-05, 2.3228965801536106e-05, 2.2241309125092812e-05, 2.1277093765093014e-05, 2.031402436841745e-05, 2.5657338483142667e-05, 2.4725861294427887e-05, 2.4976741769933142e-05, 2.5458728487137705e-05, 8.85275294422172e-05, 7.398161687888205e-05, 5.796838377136737e-05, 4.402325430419296e-05, 4.0405331674264744e-05, 3.518477751640603e-05, 2.890764517360367e-05, 2.4700053472770378e-05, 1.3025312910031062e-05, 1.180571416625753e-05, 1.1266798537690192e-05, 1.1179407920280937e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00011513973367982544, "first_to_last_ratio": 1.1689761361783517, "imbalance_abs_log_ratio": 0.15612826840504843, "last_quartile_mean": 9.849622256297152e-05, "mean": 0.00011409382295823889, "normalized": [ 1.087599643787199, 1.2332404432021893, 1.1325955558216252, 1.0348100003758545, 0.9846631201892967, 0.9329597002283465, 0.8753802278299959, 0.7920881953113688, 0.705656362899583, 0.6479209049109275, 0.6172280934926675, 0.5959629720755263, 0.6485619380890737, 0.616624813163851, 0.5952030046676504, 0.5686106501058988, 0.6801552059812119, 0.6508138445342289, 0.6710903132095474, 0.7219939084470286, 2.8256801162653837, 2.4931864357424143, 2.15835309890828, 1.8232898013061003, 1.6443057108717645, 1.5033517491402877, 1.1720956223235564, 0.8974197290603315, 0.48941489739710603, 0.4307568860388074, 0.3896112863482277, 0.3793757682746689 ], "peak_layer": 21, "peak_normalized": 2.8256801162653837, "population_cv": 0.5949861940504896, "rest_mean": 8.897283448647984e-05, "spike_contrast": 2.80700465650208, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00024974716070573776, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 0.00012408840120770037, 0.00014070511679165065, 0.00012922215682920069, 0.00011806542897829786, 0.00011234397970838472, 0.0001064449388650246, 9.987547673517838e-05, 9.037237032316625e-05, 8.05110321380198e-05, 7.392377301584929e-05, 7.042191282380372e-05, 6.799569382565096e-05, 7.399691094178706e-05, 7.035308226477355e-05, 6.790898623876274e-05, 6.487496284535155e-05, 7.76015076553449e-05, 7.425383955705911e-05, 7.656725938431919e-05, 8.237504516728222e-05, 0.0003223926469217986, 0.00028445717180147767, 0.0002462547563482076, 0.00020802610379178077, 0.00018760512466542423, 0.00017152314831037074, 0.00013372887042351067, 0.00010239004768664017, 5.583921665675007e-05, 4.9146699893753976e-05, 4.445224112714641e-05, 4.328443174017593e-05 ] } } }, "post_mlp_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "6637e7e2c52ffc1501509e8cf074d32495c0b5360809f4fcf8995d260c2092c2", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.09451264888048172, 0.2031717449426651, 0.22856928408145905, 0.2430499792098999, 0.08243431895971298, 0.09709372371435165, 0.11282631754875183, 0.13645486533641815, 0.05442482605576515, 0.0885990634560585, 0.12215005606412888, 0.15440259873867035, 0.060873307287693024, 0.09471380710601807, 0.1260664463043213, 0.163865864276886, 0.08853588253259659, 0.12131853401660919, 0.15660560131072998, 0.2097783088684082, 0.04455159604549408, 0.07411240041255951, 0.10008715838193893, 0.13061410188674927, 0.04960336163640022, 0.07874798029661179, 0.11358420550823212, 0.1580832600593567, 0.06955241411924362, 0.13302887976169586, 0.22457914054393768, 0.32859006524086 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.3873141369913355e-05, "first_to_last_ratio": 1.3885721694739201, "imbalance_abs_log_ratio": 0.32827600298864523, "last_quartile_mean": 9.990940100124135e-06, "mean": 1.2854364840109156e-05, "normalized": [ 1.1519312840854947, 1.3311758992301272, 1.2167315437597699, 1.1117686172453958, 1.058772632613286, 0.9928324590403058, 0.927417632168206, 0.8434123244463263, 0.7336779055664342, 0.6915156665898262, 0.6596055773251193, 0.6376390806056413, 0.6952626444464219, 0.6629130327970861, 0.6345504665270513, 0.6122283613875565, 0.7648834288449639, 0.7297709534484661, 0.7866551404509706, 0.8716621799696268, 2.8889843133608784, 2.338532478770988, 1.899676736110628, 1.5404711947102903, 1.5315734843108408, 1.1978159493624418, 0.9082426544704836, 0.8160818433380076, 0.4896280426071342, 0.4364018959793389, 0.4210420673902388, 0.4171425090406535 ], "peak_layer": 21, "peak_normalized": 2.8889843133608784, "population_cv": 0.5419273520370343, "rest_mean": 1.037907206948687e-05, "spike_contrast": 2.5263285220415503, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.6220945801469498e-05, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 1.4807344996370375e-05, 1.7111420675064437e-05, 1.5640311175957322e-05, 1.429107942385599e-05, 1.3609849702334031e-05, 1.2762230653606821e-05, 1.1921364603040274e-05, 1.0841529729077592e-05, 9.430963473278098e-06, 8.888994670996908e-06, 8.478810741507914e-06, 8.196445378416684e-06, 8.937159691413399e-06, 8.521325980836991e-06, 8.15674320620019e-06, 7.869806722737849e-06, 9.832090654526837e-06, 9.380742085340898e-06, 1.0111952178704087e-05, 1.120466367865447e-05, 3.713605838129297e-05, 3.00603496725671e-05, 2.4419137844233774e-05, 1.98017787624849e-05, 1.9687404346768744e-05, 1.539716322440654e-05, 1.1674882443912793e-05, 1.0490213753655553e-05, 6.293857495620614e-06, 5.609669187833788e-06, 5.4122283472679555e-06, 5.3621020015270915e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 5.706737192667788e-05, "first_to_last_ratio": 1.4278557447018398, "imbalance_abs_log_ratio": 0.3561738396982594, "last_quartile_mean": 3.996718305643299e-05, "mean": 5.182250379220932e-05, "normalized": [ 1.1683241108409874, 1.3693593420455452, 1.2478029467661282, 1.1320859203517522, 1.0862154844943164, 1.0136862423485675, 0.9398593392770029, 0.8523331101914637, 0.7376332926932397, 0.6943335382230071, 0.6618887156422048, 0.6379567874173152, 0.6961101086882286, 0.6619092142621851, 0.6320190672968202, 0.6099299790483219, 0.7532799174046017, 0.7156061917099562, 0.758449009476584, 0.8619189528616946, 2.863607509132394, 2.3232710469251585, 1.8820910428129736, 1.530471615417879, 1.517753064129064, 1.1793980690808934, 0.8973983434078874, 0.8090471852696426, 0.4887601793176453, 0.43698385938136236, 0.4222309955613731, 0.41828581852380436 ], "peak_layer": 21, "peak_normalized": 2.863607509132394, "population_cv": 0.5384894543542103, "rest_mean": 4.200080675892187e-05, "spike_contrast": 2.4966108002124767, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00010485966777196154, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 6.054548066458665e-05, 7.096362969605252e-05, 6.466427294071764e-05, 5.866752690053545e-05, 5.6290406064363196e-05, 5.253175913821906e-05, 4.870586417382583e-05, 4.417003583512269e-05, 3.822600410785526e-05, 3.59821024176199e-05, 3.430073047638871e-05, 3.306051803519949e-05, 3.607416874729097e-05, 3.430179276620038e-05, 3.275281051173806e-05, 3.160809865221381e-05, 3.903685137629509e-05, 3.708450458361767e-05, 3.9304726669797674e-05, 4.4666798203252256e-05, 0.00014839931100141257, 0.00012039772263960913, 9.753467020345852e-05, 7.931287109386176e-05, 7.865376392146572e-05, 6.111936090746894e-05, 4.65054290543776e-05, 4.1926850826712325e-05, 2.5328776246169582e-05, 2.2645597709924914e-05, 2.1881067368667573e-05, 2.167661841667723e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.05060791550204158, "first_to_last_ratio": 1.4278557661609772, "imbalance_abs_log_ratio": 0.3561738547271846, "last_quartile_mean": 0.035443296656012535, "mean": 0.045956714369822294, "normalized": [ 1.1683241037091443, 1.3693593062308924, 1.2478029694871473, 1.1320859320845278, 1.0862154618577222, 1.013686266934737, 0.9398592878142475, 0.8523331062206766, 0.7376333014160883, 0.6943335139035525, 0.6618887468442457, 0.6379567875276879, 0.6961101245379498, 0.6619092147086973, 0.632019080315496, 0.6099299583472245, 0.75327990973307, 0.7156062315279661, 0.7584489979720741, 0.861918956956209, 2.863607441781645, 2.3232710596995285, 1.8820912266878924, 1.5304716351616434, 1.5177530257827039, 1.179397988455519, 0.8973983179704046, 0.8090471801133733, 0.48876020079385396, 0.43698383973821214, 0.4222310083457578, 0.41828581734010906 ], "peak_layer": 21, "peak_normalized": 2.863607441781645, "population_cv": 0.538489454771283, "rest_mean": 0.037246735076661465, "spike_contrast": 2.496610840158122, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.09299060255289078, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 0.05369233712553978, 0.06293125450611115, 0.05734492465853691, 0.05202694982290268, 0.04991889372467995, 0.04658569023013115, 0.043192844837903976, 0.03917042911052704, 0.033899202942848206, 0.031909286975860596, 0.030418232083320618, 0.029318397864699364, 0.031990934163331985, 0.03041917271912098, 0.02904552035033703, 0.028030376881361008, 0.03461826965212822, 0.03288691118359566, 0.03485582396388054, 0.03961096331477165, 0.1316019892692566, 0.10676990449428558, 0.08649472892284393, 0.07033544778823853, 0.06975094228982925, 0.05420125648379326, 0.04124147817492485, 0.037181150168180466, 0.02246181294322014, 0.020082341507077217, 0.019404349848628044, 0.019223041832447052 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 2.187229483752162e-06, "first_to_last_ratio": 0.843721752674449, "imbalance_abs_log_ratio": 0.16993251566757275, "last_quartile_mean": 2.592358768538361e-06, "mean": 2.5683487088201673e-06, "normalized": [ 1.191975321874382, 0.9792375219295126, 0.9240659028379828, 0.9165026822465744, 0.7327411746199551, 0.7055071798348216, 0.68634443005073, 0.6764996809984228, 0.5799709708281261, 0.5635565193402196, 0.5596940376847175, 0.5639570693866405, 0.5739412518179093, 0.5756289711426894, 0.5747974170292999, 0.5782423953100819, 0.652186394989175, 0.6637611802901628, 0.6844201166273622, 0.7655503355998897, 2.843845838013164, 2.5945154076809365, 2.2989812705463573, 2.039289390897498, 1.9644018920207635, 1.5269105875005642, 1.1791879088220734, 1.0934639133198125, 0.6272666844428548, 0.5710249694984988, 0.5563916798315591, 0.5561399029872623 ], "peak_layer": 21, "peak_normalized": 2.843845838013164, "population_cv": 0.6360528130844023, "rest_mean": 1.9271144327393374e-06, "spike_contrast": 3.129556655898081, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.031013799656648e-06, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 3.061408278881572e-06, 2.515023425075924e-06, 2.3733234684186755e-06, 2.3538984805782093e-06, 1.8819348497345345e-06, 1.8119884543921216e-06, 1.762771830726706e-06, 1.7374870822095545e-06, 1.4895676940795965e-06, 1.4474096587946406e-06, 1.43748945902189e-06, 1.4484384109891835e-06, 1.474081273045158e-06, 1.4784159247938078e-06, 1.4762802038603695e-06, 1.4851281093797297e-06, 1.675042085480527e-06, 1.7047701703631901e-06, 1.7578295228304341e-06, 1.9662002159748226e-06, 7.303987786144717e-06, 6.663620297331363e-06, 5.904585577809485e-06, 5.237606274022255e-06, 5.0452690629754215e-06, 3.921638835890917e-06, 3.0285657430795254e-06, 2.8083966299163876e-06, 1.6110395790747134e-06, 1.4665912431155448e-06, 1.429007852493669e-06, 1.4283612017607084e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 4.514497459240374e-05, "first_to_last_ratio": 1.6555269986325893, "imbalance_abs_log_ratio": 0.5041193863183254, "last_quartile_mean": 2.7269246970718086e-05, "mean": 3.9283020413449776e-05, "normalized": [ 1.2428521215956514, 1.4224732380710257, 1.3076049249463846, 1.1908965547141857, 1.1240104927458778, 1.0512648458306981, 0.9724745802930264, 0.8822121342574466, 0.7835766828764287, 0.7378119649346744, 0.7017902701963611, 0.6755581812134592, 0.7400830264097682, 0.701223315160251, 0.6687269842506862, 0.6449060748365787, 0.8086534690047941, 0.7678965143140377, 0.7972719531557511, 0.8841907350981015, 2.849012586533336, 2.2825637536620915, 1.7952254297971346, 1.4143290706697607, 1.3505415355877917, 1.056910410344958, 0.832807309381792, 0.7560735292299187, 0.4172799715855694, 0.3864473693959841, 0.37654214074567177, 0.3767888291608037 ], "peak_layer": 21, "peak_normalized": 2.849012586533336, "population_cv": 0.5250931691969987, "rest_mean": 3.245698109013675e-05, "spike_contrast": 2.345986293299445, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 7.61436327593401e-05, "top_five_layers": [ 21, 22, 23, 2, 24 ], "top_five_spike_overlap": 4 }, "values": [ 4.882298526354134e-05, 5.587904524873011e-05, 5.136667095939629e-05, 4.678201366914436e-05, 4.415452713146806e-05, 4.1296858398709446e-05, 3.820173878921196e-05, 3.465595727902837e-05, 3.078125882893801e-05, 2.8983482479816303e-05, 2.7568441510084085e-05, 2.6537965823081322e-05, 2.9072696634102613e-05, 2.754616980382707e-05, 2.6269615773344412e-05, 2.533385850256309e-05, 3.17663507303223e-05, 3.0165294447215274e-05, 3.1319250410888344e-05, 3.4733682696241885e-05, 0.00011191781959496439, 8.966599853010848e-05, 7.052187720546499e-05, 5.5559117754455656e-05, 5.3053350711707026e-05, 4.1518633224768564e-05, 3.271518653491512e-05, 2.970085188280791e-05, 1.6392017641919665e-05, 1.518081990070641e-05, 1.4791712601436302e-05, 1.4801403267483693e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 3.469784724074998e-05, "first_to_last_ratio": 1.7508907324210148, "imbalance_abs_log_ratio": 0.5601246483988392, "last_quartile_mean": 1.9817254496956593e-05, "mean": 2.9772389552817913e-05, "normalized": [ 1.2825657703659485, 1.4312331552120288, 1.325820640442136, 1.2221823686225868, 1.130955243942597, 1.0564145565086833, 0.9757410003641811, 0.8985840377367545, 0.8214467481888739, 0.7702253806339946, 0.7297568784639149, 0.7040532057825352, 0.7682184821342395, 0.7314383768963997, 0.6969099228187515, 0.6697241707472397, 0.8568233441617242, 0.8208320560978474, 0.8328912261837936, 0.8882556709751984, 2.8316971252422305, 2.2395765326860197, 1.6996117178665944, 1.2900403186629539, 1.2152436368088633, 0.9720044593739222, 0.8310137912106149, 0.7782207156702814, 0.3990111711474371, 0.3786791961781316, 0.37596186733999515, 0.3748672315335271 ], "peak_layer": 21, "peak_normalized": 2.8316971252422305, "population_cv": 0.507296052444767, "rest_mean": 2.505713847414073e-05, "spike_contrast": 2.204351683440534, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.5234745377674696e-05, "top_five_layers": [ 21, 22, 23, 2, 3 ], "top_five_spike_overlap": 3 }, "values": [ 3.818504774244502e-05, 4.2611231037881225e-05, 3.94728485844098e-05, 3.638728958321735e-05, 3.367124008946121e-05, 3.145198570564389e-05, 2.9050141165498644e-05, 2.675299401744269e-05, 2.4456432583974674e-05, 2.2931450075702742e-05, 2.1726606064476073e-05, 2.096134630846791e-05, 2.2871699911775067e-05, 2.1776668290840462e-05, 2.0748673705384135e-05, 1.9939288904424757e-05, 2.5509678380331025e-05, 2.44381317315856e-05, 2.4797162041068077e-05, 2.644549385877326e-05, 8.43063899083063e-05, 6.667754496447742e-05, 5.06015021528583e-05, 3.840758290607482e-05, 3.618070695665665e-05, 2.8938895411556587e-05, 2.4741266315686516e-05, 2.3169490305008367e-05, 1.1879516023327596e-05, 1.127418454416329e-05, 1.1193283171451185e-05, 1.1160693247802556e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00011355107744748238, "first_to_last_ratio": 1.3380154347456796, "imbalance_abs_log_ratio": 0.2911874973238657, "last_quartile_mean": 8.486529713991331e-05, "mean": 0.00010590487272565952, "normalized": [ 1.14177249304275, 1.318239805267508, 1.2190343862407116, 1.1130801571737208, 1.0501716457006294, 0.9937227743286174, 0.9154089043884999, 0.8261602336216923, 0.7161833889440711, 0.6746260480574641, 0.6498212581625004, 0.6259761084931398, 0.6817894792861978, 0.6448396206511546, 0.6202148320012568, 0.5943358056681826, 0.7276123000587434, 0.6905927768196939, 0.7174064355718048, 0.8068892453627287, 2.8834514867746446, 2.3459559521323445, 1.9722659474769593, 1.659767709595541, 1.6192051847503666, 1.2622818151535096, 0.9611298389601171, 0.8706981752777411, 0.4632426508072287, 0.41992689699434765, 0.4080288833982593, 0.4061677598378732 ], "peak_layer": 21, "peak_normalized": 2.8834514867746446, "population_cv": 0.5599097168924846, "rest_mean": 8.44075709917686e-05, "spike_contrast": 2.6299809303873802, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00022199030208867044, "top_five_layers": [ 21, 22, 23, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 0.00012091927055735141, 0.00013960801879875362, 0.00012910168152302504, 0.00011788061237893999, 0.00011121829447802156, 0.0001052400839398615, 9.69462635111995e-05, 8.749439439270645e-05, 7.584731065435335e-05, 7.14461857569404e-05, 6.881923764012754e-05, 6.62939200992696e-05, 7.220482802949846e-05, 6.82916579535231e-05, 6.56837728456594e-05, 6.294305785559118e-05, 7.70576880313456e-05, 7.313714013434947e-05, 7.597683725180104e-05, 8.545350283384323e-05, 0.00030537156271748245, 0.00024844816653057933, 0.00020887257414869964, 0.00017577748803887516, 0.00017148171900771558, 0.0001336817949777469, 0.00010178833326790482, 9.221117943525314e-05, 4.905965397483669e-05, 4.447230458026752e-05, 4.321224696468562e-05, 4.30151449108962e-05 ] } } }, "pre_attention_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "9f622711336f6114d5391a2fcd50030068888edcdff338c4d870b49ef8b7290b", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.05595625191926956, 0.0632273480296135, 0.11160287261009216, 0.12218338996171951, 0.12365243583917618, 0.0926642045378685, 0.093480683863163, 0.08749273419380188, 0.09927791357040405, 0.07820390164852142, 0.07669360190629959, 0.07840896397829056, 0.08920575678348541, 0.06850729882717133, 0.07440610229969025, 0.07867594808340073, 0.09058429300785065, 0.07234266400337219, 0.07771008461713791, 0.08416424691677094, 0.08947562426328659, 0.056279782205820084, 0.057985786348581314, 0.06238175556063652, 0.06515856087207794, 0.05041429027915001, 0.05083387345075607, 0.05600840970873833, 0.06128782033920288, 0.056301049888134, 0.06424970924854279, 0.06844019144773483 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 9.178191703540506e-06, "first_to_last_ratio": 2.0927096232778837, "imbalance_abs_log_ratio": 0.7384596967524485, "last_quartile_mean": 4.385793232586366e-06, "mean": 9.837752486419049e-06, "normalized": [ 1.3396520059806467, 1.2909822759863991, 0.8248929730174986, 0.8734689592802515, 0.4594300774035242, 0.8015983028871521, 0.902358022333612, 0.9712665986901348, 0.6517279620702138, 0.5032471363266136, 0.3886238838550298, 0.3309759496712648, 0.3194854781922982, 0.5459377912807375, 0.43221827830345944, 0.4611789434475917, 1.0400100603479894, 1.0312967006898324, 1.4985429159426815, 1.6347346226131838, 2.6918571666331577, 4.297443405515621, 2.6798531616309957, 2.4627171634441143, 1.010352002858167, 0.8524077751550129, 0.3921416774715023, 0.37035603642720544, 0.29698661283582345, 0.19544726726737693, 0.19990393116510585, 0.24890486127580272 ], "peak_layer": 22, "peak_normalized": 4.297443405515621, "population_cv": 0.8934604862364398, "rest_mean": 6.871042353816813e-06, "spike_contrast": 3.763328163463758, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.5857987202471124e-05, "top_five_layers": [ 22, 21, 23, 24, 20 ], "top_five_spike_overlap": 4 }, "values": [ 1.3179164852772374e-05, 1.2700364095508121e-05, 8.115092896332499e-06, 8.592971425969154e-06, 4.5197593863122165e-06, 7.88592569733737e-06, 8.877174877852667e-06, 9.555080396239646e-06, 6.411538379325066e-06, 4.950820766680408e-06, 3.823185579676647e-06, 3.2560594718233915e-06, 3.1430190574610606e-06, 5.370800863602199e-06, 4.252056442055618e-06, 4.536964297585655e-06, 1.0231361557089258e-05, 1.014564168144716e-05, 1.4742294297320768e-05, 1.6082114598248154e-05, 2.6481824534130283e-05, 4.227718454785645e-05, 2.636373210407328e-05, 2.4227601898019202e-05, 9.939592928276397e-06, 8.385776709474158e-06, 3.8577927625738084e-06, 3.6434710182220442e-06, 2.9216807888587937e-06, 1.922761839523446e-06, 1.9666053958644625e-06, 2.4486644178978167e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 3.4749664109767764e-05, "first_to_last_ratio": 2.002663558914822, "imbalance_abs_log_ratio": 0.6944780739856734, "last_quartile_mean": 1.735172338612756e-05, "mean": 3.858046457594355e-05, "normalized": [ 0.8375866313273315, 1.2899140602905792, 0.8694344042805302, 0.9158393863926719, 0.4754536346534661, 0.8638099380048725, 0.9821647500595908, 0.9714468918970042, 0.6838803886511606, 0.5360010655021746, 0.3851151654093992, 0.3481231986646324, 0.3212694297422333, 0.5688836414280135, 0.4682021401391152, 0.46212538368136635, 1.0036599670842112, 1.0844213697059686, 1.4666068093127032, 1.6452318694807484, 2.683098703810988, 4.385873690297744, 2.767792621757238, 2.3860317964922024, 1.0350167413971205, 0.871092691603006, 0.39147360723073643, 0.341414306210179, 0.30371057059916196, 0.20069739154416044, 0.20302718339269, 0.25160056995699964 ], "peak_layer": 22, "peak_normalized": 4.385873690297744, "population_cv": 0.8995046127848129, "rest_mean": 2.6780824454261972e-05, "spike_contrast": 3.8198421190406635, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00010229852123302407, "top_five_layers": [ 22, 23, 21, 24, 20 ], "top_five_spike_overlap": 4 }, "values": [ 3.2314481359208e-05, 4.9765483709052205e-05, 3.354318323545158e-05, 3.533350900397636e-05, 1.8343222109251656e-05, 3.332618871354498e-05, 3.7892372347414494e-05, 3.7478872400242835e-05, 2.6384423108538613e-05, 2.0679170120274648e-05, 1.4857921996735968e-05, 1.3430754734145012e-05, 1.239472385350382e-05, 2.194779517594725e-05, 1.8063456082018092e-05, 1.7829011994763277e-05, 3.8721667806385085e-05, 4.183748023933731e-05, 5.658237205352634e-05, 6.34738098597154e-05, 0.00010351519449613988, 0.00016920904454309493, 0.00010678272519726306, 9.205421520164236e-05, 3.9931426726980135e-05, 3.3607160730753094e-05, 1.5103233636182267e-05, 1.3171922546462156e-05, 1.171729491034057e-05, 7.742998604953755e-06, 7.832883056835271e-06, 9.706866876513232e-06 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.030816350365057588, "first_to_last_ratio": 2.002663649499559, "imbalance_abs_log_ratio": 0.6944781192178017, "last_quartile_mean": 0.015387681487482041, "mean": 0.03421354205056559, "normalized": [ 0.8375866167785538, 1.2899141113192616, 0.8694344424609419, 0.915839393595925, 0.475453642772802, 0.8638099562451347, 0.9821647993180688, 0.9714468517291538, 0.6838804035566444, 0.5360010365403223, 0.3851151782472681, 0.3481232059212625, 0.321269430974011, 0.5688836365906816, 0.46820216481780275, 0.4621253765317527, 1.0036599271213709, 1.0844213253993487, 1.4666067752392928, 1.645231903565359, 2.6830987807805085, 4.385873655623207, 2.767792506758732, 2.3860319203463485, 1.0350167426483041, 0.8710926293236593, 0.3914735937130686, 0.34141432168112545, 0.3037105506770517, 0.20069737952502648, 0.20302717344287544, 0.25160056675513426 ], "peak_layer": 22, "peak_normalized": 4.385873655623207, "population_cv": 0.8995046133337352, "rest_mean": 0.023749503048343792, "spike_contrast": 3.8198421448186797, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.09071935266256333, "top_five_layers": [ 22, 23, 21, 24, 20 ], "top_five_spike_overlap": 4 }, "values": [ 0.02865680493414402, 0.0441325306892395, 0.02974643185734749, 0.03133410960435867, 0.016266953200101852, 0.02955399826169014, 0.03360333666205406, 0.03323663771152496, 0.02339797094464302, 0.01833849400281906, 0.013176154345273972, 0.01191052794456482, 0.010991765186190605, 0.019463524222373962, 0.016018854454159737, 0.015810946002602577, 0.03433876112103462, 0.03710189461708069, 0.050177812576293945, 0.05628921091556549, 0.0917983129620552, 0.15005627274513245, 0.09469598531723022, 0.08163460344076157, 0.035411588847637177, 0.029803164303302765, 0.013393698260188103, 0.011680993251502514, 0.010391013696789742, 0.006866568233817816, 0.006946278735995293, 0.008608146570622921 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 2.8313342426145027e-06, "first_to_last_ratio": 0.9841100436889928, "imbalance_abs_log_ratio": 0.01601755516546022, "last_quartile_mean": 2.87705044854647e-06, "mean": 2.8823285838086576e-06, "normalized": [ 3.860850182514526, 1.3779145113400497, 0.37073650468044217, 0.4054583128312923, 0.3411145989186331, 0.46814184382330143, 0.46288850930954656, 0.5713590261909561, 0.3765635727895922, 0.3810005221832118, 0.35155981621084625, 0.393783350864445, 0.33878649322165827, 0.47478509983905887, 0.4584607500663275, 0.5226177027680562, 0.4037194811557118, 0.6360221076907124, 0.6562906885789336, 0.9724019253458765, 1.6772106521914107, 3.0942458588650434, 2.9339099985078207, 2.4848281321882983, 1.7824259625811416, 1.6267190981510171, 1.2975773719544916, 1.1633314414023308, 0.5790326834473365, 0.5326514143129327, 0.49977516979896786, 0.503837216276031 ], "peak_layer": 1, "peak_normalized": 3.860850182514526, "population_cv": 0.9094699254837627, "rest_mean": 2.1379810404040554e-06, "spike_contrast": 3.2281882709722924, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.90180531819351e-06, "top_five_layers": [ 1, 22, 23, 24, 25 ], "top_five_spike_overlap": 4 }, "values": [ 1.112823883886449e-05, 3.971602382080164e-06, 1.0685844245017506e-06, 1.1686640846164664e-06, 9.83204358817602e-07, 1.3493386177287903e-06, 1.334196781499486e-06, 1.6468444528072723e-06, 1.0853799494725536e-06, 1.0981686955346959e-06, 1.0133109071830404e-06, 1.1350130080245435e-06, 9.764939932210837e-07, 1.3684866644325666e-06, 1.3214345244705328e-06, 1.5063559430927853e-06, 1.163652200375509e-06, 1.8332247009311686e-06, 1.8916454109785263e-06, 2.8027818643749924e-06, 4.834272203879664e-06, 8.918633284338284e-06, 8.456492651021108e-06, 7.16209115125821e-06, 5.137537300470285e-06, 4.688738954428118e-06, 3.74004434888775e-06, 3.3531034659972647e-06, 1.6689624544596882e-06, 1.535276396680274e-06, 1.4405162573893904e-06, 1.4522244100589887e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 2.3673139480706595e-05, "first_to_last_ratio": 2.257300425332895, "imbalance_abs_log_ratio": 0.8141695972328834, "last_quartile_mean": 1.0487367660516611e-05, "mean": 2.6041475749138954e-05, "normalized": [ 0.6771788519354571, 1.2979325532283608, 0.9303271057055151, 0.9639489120456851, 0.5271438854731791, 0.9092278846604233, 1.028243959121235, 0.9384386122687313, 0.6892881365249813, 0.5698927083153338, 0.44101071224628413, 0.3889922048793135, 0.37294517808508393, 0.6424365166247796, 0.5244231346818359, 0.4778644993700143, 1.0150499086102556, 1.1538261154181413, 1.4466435534537925, 1.6137729430394836, 2.6468294071482537, 4.220741079232897, 2.9670008977788065, 2.335098275492907, 0.997666272782542, 0.8091786594449493, 0.3691954363618722, 0.3097208222153953, 0.23815117281925108, 0.1573714206968098, 0.14298175278225358, 0.19747742755617637 ], "peak_layer": 22, "peak_normalized": 4.220741079232897, "population_cv": 0.8898631795325547, "rest_mean": 1.816408757804311e-05, "spike_contrast": 3.7755473033479423, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.857937187305651e-05, "top_five_layers": [ 22, 23, 21, 24, 20 ], "top_five_spike_overlap": 4 }, "values": [ 1.7634736650506966e-05, 3.380007910891436e-05, 2.4227090761996806e-05, 2.5102652216446586e-05, 1.3727604709856678e-05, 2.3677635908825323e-05, 2.6776990125654265e-05, 2.443832636345178e-05, 1.7950080291484483e-05, 1.4840847143204883e-05, 1.1484569768072106e-05, 1.0129931069968734e-05, 9.712042810861021e-06, 1.6729994968045503e-05, 1.3656752344104461e-05, 1.2444296771718655e-05, 2.643339757923968e-05, 3.004733480338473e-05, 3.767273301491514e-05, 4.202502896077931e-05, 6.892734381835908e-05, 0.00010991432645823807, 7.726508192718029e-05, 6.080940511310473e-05, 2.5980702048400417e-05, 2.1072206436656415e-05, 9.614394002710469e-06, 8.065587280725595e-06, 6.201807991601527e-06, 4.098184035683516e-06, 3.723455847648438e-06, 5.142603640706511e-06 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 1.5893040995251795e-05, "first_to_last_ratio": 2.4728739688838175, "imbalance_abs_log_ratio": 0.9053810243954281, "last_quartile_mean": 6.426951472349174e-06, "mean": 1.7339248245207273e-05, "normalized": [ 0.5512149265120582, 1.2985625489006958, 0.9783452688543264, 0.9971647574760376, 0.5962146785243507, 0.9290000522923118, 1.0861697535899135, 0.8960754411880637, 0.726825196328683, 0.6059411861934751, 0.5099587939140718, 0.44057324634523176, 0.4391978251197769, 0.7191858964012756, 0.5811495621714656, 0.4881322826391351, 1.0011626163335512, 1.1990369599246455, 1.4118886136322046, 1.554688514867236, 2.6196553178384634, 3.9391535171303675, 3.1321858798359785, 2.3332437535315793, 0.9448004540825558, 0.7676606562766751, 0.3665181536820326, 0.2919617681864151, 0.18930565608900826, 0.13559488549785165, 0.11350226730002422, 0.15592956934053803 ], "peak_layer": 22, "peak_normalized": 3.9391535171303675, "population_cv": 0.8670378924383861, "rest_mean": 1.2221576239594637e-05, "spike_contrast": 3.679940802546367, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.49746770755155e-05, "top_five_layers": [ 22, 23, 21, 24, 20 ], "top_five_spike_overlap": 4 }, "values": [ 9.55765244725626e-06, 2.2516098397318274e-05, 1.6963771486189216e-05, 1.7290087271248922e-05, 1.0337914318370167e-05, 1.610816252650693e-05, 1.8833366993931122e-05, 1.5537274521193467e-05, 1.2602602510014549e-05, 1.0506564649404027e-05, 8.842302122502588e-06, 7.639208888576832e-06, 7.615360118506942e-06, 1.2470142792153638e-05, 1.0076696526084561e-05, 8.463846825179644e-06, 1.735940713842865e-05, 2.0790399503312074e-05, 2.4481087166350335e-05, 2.6957130103255622e-05, 4.542285387287848e-05, 6.830196070950478e-05, 5.430974852060899e-05, 4.0456692659063265e-05, 1.638212961552199e-05, 1.3310658687260002e-05, 6.355149253067793e-06, 5.062397576693911e-06, 3.282417765149148e-06, 2.351113380427705e-06, 1.968043989108992e-06, 2.7037015115638496e-06 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 7.000305004112306e-05, "first_to_last_ratio": 2.193525307703265, "imbalance_abs_log_ratio": 0.7855099791473917, "last_quartile_mean": 3.191349094322504e-05, "mean": 7.690838336316119e-05, "normalized": [ 0.8923720562068232, 1.3111973164248405, 0.8460717362092315, 0.9445716926262052, 0.4741848629196519, 0.8656844065975011, 0.9924506003335006, 0.9551754577499273, 0.6234358560310297, 0.5152936103919111, 0.3846693562402596, 0.34526537659024253, 0.30979982474526946, 0.5791064055325631, 0.47493901082375733, 0.4350256040989574, 1.011874248892453, 1.1362510856108166, 1.416617477431854, 1.6892273518644711, 2.692715672842754, 4.577499824936365, 2.8907687343273323, 2.31616526773872, 1.0250323630610032, 0.8340050859269844, 0.36589411026523266, 0.29901912364743294, 0.2532838824218693, 0.16506308776319403, 0.15276161475417316, 0.22457789499367237 ], "peak_layer": 22, "peak_normalized": 4.577499824936365, "population_cv": 0.9353823752601559, "rest_mean": 5.269026995072771e-05, "spike_contrast": 3.9416422801499356, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.000207686195790302, "top_five_layers": [ 22, 23, 21, 24, 20 ], "top_five_spike_overlap": 4 }, "values": [ 6.863089220132679e-05, 0.00010084206587634981, 6.507000944111496e-05, 7.264548185048625e-05, 3.646879122243263e-05, 6.657838821411133e-05, 7.632777123944834e-05, 7.346100028371438e-05, 4.7947443817975e-05, 3.9630398532608524e-05, 2.9584298317786306e-05, 2.6553801944828592e-05, 2.3826203687349334e-05, 4.453813744476065e-05, 3.652679151855409e-05, 3.34571159328334e-05, 7.782161264913157e-05, 8.738723408896476e-05, 0.00010894976003328338, 0.00012991574476473033, 0.00020709240925498307, 0.0003520481113810092, 0.00022232435003388673, 0.00017813252634368837, 7.883358193794265e-05, 6.414198287529871e-05, 2.8140324502601288e-05, 2.2997077394393273e-05, 1.9479653929010965e-05, 1.2694735232798848e-05, 1.174864883068949e-05, 1.7271922843065113e-05 ] } } }, "pre_mlp_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "150a36c8e5b01ef896c4498cdc7a2002d1b68d6cfc87a698c943b18cb0623a93", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.047033343464136124, 0.060984931886196136, 0.09079429507255554, 0.08531255275011063, 0.0786692351102829, 0.08076046407222748, 0.08904207497835159, 0.08403915911912918, 0.06012389436364174, 0.06841155141592026, 0.07131828367710114, 0.07358616590499878, 0.06597626209259033, 0.07235056161880493, 0.07261744141578674, 0.07352335751056671, 0.07520031183958054, 0.07378771901130676, 0.07715436071157455, 0.08649503439664841, 0.04556623473763466, 0.04546353965997696, 0.05167657509446144, 0.058827146887779236, 0.04798740893602371, 0.046526361256837845, 0.051699329167604446, 0.059578344225883484, 0.05090610682964325, 0.05821637064218521, 0.06188955530524254, 0.06796324998140335 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 4.364372159670893e-06, "first_to_last_ratio": 0.2681115118992595, "imbalance_abs_log_ratio": 1.3163522958360352, "last_quartile_mean": 1.627819756322424e-05, "mean": 1.0259845446114468e-05, "normalized": [ 1.0134993414800566, 0.522056581911947, 0.2644061262673726, 0.3833634649429024, 0.2440106441209935, 0.2461172286895863, 0.27872130983193855, 0.4508957935090263, 0.4833194134691065, 0.39025737906993885, 0.3708598519154414, 0.36981229946243677, 0.42398498232424564, 0.41928891386718925, 0.4397458055599365, 0.45615006609958, 0.4114730365687318, 0.5108641768312265, 0.48188059954330154, 0.7614115358485232, 2.335493650100753, 2.8449380275824225, 2.7201337953566025, 2.484573069357449, 2.4148997615580035, 2.4501695400243877, 2.1889991494613, 1.6914475251239836, 1.1580207584918483, 1.0537166329047938, 0.8749014303613178, 0.8605881083636573 ], "peak_layer": 22, "peak_normalized": 2.8449380275824225, "population_cv": 0.8504187150143712, "rest_mean": 7.295875539731006e-06, "spike_contrast": 3.6000179550148346, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.6265282940585166e-05, "top_five_layers": [ 22, 23, 24, 26, 25 ], "top_five_spike_overlap": 4 }, "values": [ 1.0398346603324171e-05, 5.356219844543375e-06, 2.7127659905090695e-06, 3.933249900001101e-06, 2.503511495888233e-06, 2.525124727981165e-06, 2.8596375614142744e-06, 4.626121153705753e-06, 4.9587824832997285e-06, 4.00398039346328e-06, 3.804964762821328e-06, 3.7942170365568018e-06, 4.350020390120335e-06, 4.301839453546563e-06, 4.511724000622053e-06, 4.680029178416589e-06, 4.221649760438595e-06, 5.241387498244876e-06, 4.944020474795252e-06, 7.811964678694494e-06, 2.3961803890415467e-05, 2.9188624466769397e-05, 2.7908152333111502e-05, 2.5491335691185668e-05, 2.4776498321443796e-05, 2.5138360797427595e-05, 2.2458792955148965e-05, 1.735399018798489e-05, 1.1881114005518612e-05, 1.081096979760332e-05, 8.976353456091601e-06, 8.829500984575134e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 1.76496694166417e-05, "first_to_last_ratio": 0.27059749062388033, "imbalance_abs_log_ratio": 1.3071228366665688, "last_quartile_mean": 6.522480816784082e-05, "mean": 4.117927500146834e-05, "normalized": [ 1.007463472142432, 0.5209651751221482, 0.2643970395144652, 0.3829104157055737, 0.2536844256534051, 0.25437837481906395, 0.28207653195017285, 0.46296966564632774, 0.49565891190391403, 0.39385639261196637, 0.3771016062127897, 0.37530908825337306, 0.4278590182802805, 0.42183720924091433, 0.44371630887119135, 0.4598267075919064, 0.41216967133849985, 0.5079852336529339, 0.47800548121281095, 0.7481511452590028, 2.3310574965471607, 2.8352891763903005, 2.718777452615949, 2.4731685677354274, 2.4074108105775847, 2.4550622811024825, 2.180382977806297, 1.6846495469817935, 1.161664542480338, 1.0519342613419045, 0.8725636415138176, 0.8577173699237725 ], "peak_layer": 22, "peak_normalized": 2.8352891763903005, "population_cv": 0.8467657487851442, "rest_mean": 2.9335347587928278e-05, "spike_contrast": 3.5839521832647088, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00010513648303458467, "top_five_layers": [ 22, 23, 24, 26, 25 ], "top_five_spike_overlap": 4 }, "values": [ 4.148661537328735e-05, 2.1452968212543055e-05, 1.0887678399740253e-05, 1.576797330926638e-05, 1.0446540727571119e-05, 1.0475117051100824e-05, 1.1615707080636639e-05, 1.906475517898798e-05, 2.0410874640219845e-05, 1.6218720702454448e-05, 1.552877074573189e-05, 1.5454956155736e-05, 1.761892417562194e-05, 1.7370950445183553e-05, 1.8271915905643255e-05, 1.8935330444946885e-05, 1.697284824331291e-05, 2.0918463633279316e-05, 1.968391916307155e-05, 3.080832175328396e-05, 9.599125769454986e-05, 0.00011675515270326287, 0.00011195728438906372, 0.00010184328857576475, 9.913543181028217e-05, 0.0001010976848192513, 8.978659025160596e-05, 6.937264697626233e-05, 4.783650365425274e-05, 4.3317890231264755e-05, 3.593153815018013e-05, 3.5320179449627176e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.015651903115212917, "first_to_last_ratio": 0.2705974872956941, "imbalance_abs_log_ratio": 1.307122848965967, "last_quartile_mean": 0.05784201202914119, "mean": 0.03651819290826097, "normalized": [ 1.0074634186476772, 0.5209651919181703, 0.2643970221469926, 0.382910419856966, 0.2536844208302265, 0.25437835679028203, 0.2820765262777231, 0.4629696917045884, 0.4956588954855505, 0.393856398901423, 0.3771015816519749, 0.37530907952450476, 0.42785903141736564, 0.4218372178649866, 0.4437163224543097, 0.4598267050189124, 0.41216967747329053, 0.5079852485753572, 0.4780054808986325, 0.7481511542287224, 2.331057435590359, 2.835289207255424, 2.7187775380829278, 2.4731685834004287, 2.4074106915648685, 2.455062286976735, 2.180383000719517, 1.6846496365458605, 1.1616645069657927, 1.0519342829703067, 0.8725635941676015, 0.857717394092523 ], "peak_layer": 22, "peak_normalized": 2.835289207255424, "population_cv": 0.8467657515631118, "rest_mean": 0.026014879690828146, "spike_contrast": 3.5839521608577614, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.09323608428239823, "top_five_layers": [ 22, 23, 24, 26, 25 ], "top_five_spike_overlap": 4 }, "values": [ 0.036790743470191956, 0.01902470737695694, 0.009655301459133625, 0.013983196578919888, 0.00926409661769867, 0.009289437904953957, 0.010300925001502037, 0.016906816512346268, 0.018100567162036896, 0.01438292395323515, 0.013771068304777145, 0.013705609366297722, 0.01562463864684105, 0.015404732897877693, 0.01620371825993061, 0.016792040318250656, 0.01505169179290533, 0.018550703302025795, 0.017455896362662315, 0.02732112817466259, 0.08512600511312485, 0.10353963822126389, 0.09928484261035919, 0.09031564742326736, 0.08791428804397583, 0.08965443819761276, 0.07962364703416824, 0.06152036041021347, 0.042421888560056686, 0.038414739072322845, 0.03186444565653801, 0.03132228925824165 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 1.036145782506992e-06, "first_to_last_ratio": 0.23813875303874632, "imbalance_abs_log_ratio": 1.434901779225667, "last_quartile_mean": 4.35101708262664e-06, "mean": 2.601542721691885e-06, "normalized": [ 1.0445105925849996, 0.4868956651441851, 0.2653781288407282, 0.38704215458190727, 0.2311435484604591, 0.22762453648926426, 0.22559249684218238, 0.3180631703688383, 0.41371995096278646, 0.35456417415204133, 0.35816018513093856, 0.3384166475711158, 0.38409460459920464, 0.37901778296480176, 0.4237575276229331, 0.4302249635723777, 0.38747010650156044, 0.47861136725136255, 0.44057460034262785, 0.49825323727916515, 2.2524328318137035, 2.8621193651579846, 2.8149870576895797, 2.617539402453915, 2.4626516595822796, 2.6269385253668815, 2.362864616381361, 1.8136241394740558, 1.1783341687752071, 1.0893413201795874, 0.9454878584900481, 0.9005636133719171 ], "peak_layer": 22, "peak_normalized": 2.8621193651579846, "population_cv": 0.9010662730896349, "rest_mean": 1.8297776991689548e-06, "spike_contrast": 3.6993968427913813, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.76907384331571e-06, "top_five_layers": [ 22, 23, 26, 24, 25 ], "top_five_spike_overlap": 4 }, "values": [ 2.717338929869584e-06, 1.266679873879184e-06, 6.903925395818078e-07, 1.0069067002405063e-06, 6.013298161633429e-07, 5.921749561821343e-07, 5.868885182280792e-07, 8.274549259112973e-07, 1.0763101272459608e-06, 9.224138466379372e-07, 9.317690228272113e-07, 8.80405366388004e-07, 9.992385230361833e-07, 9.860309546638746e-07, 1.1024233117495896e-06, 1.1192486226718756e-06, 1.0080200354423141e-06, 1.245127918991784e-06, 1.1461736448836746e-06, 1.296227083003032e-06, 5.859800239704782e-06, 7.445925803040154e-06, 7.323309091589181e-06, 6.809640581195708e-06, 6.406693501048721e-06, 6.8340928010002244e-06, 6.147093245090218e-06, 4.718220679933438e-06, 3.0654866804979974e-06, 2.8339679829514353e-06, 2.4597270567028318e-06, 2.342854713788256e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 1.2745702917982271e-05, "first_to_last_ratio": 0.3040242464136832, "imbalance_abs_log_ratio": 1.1906478228166633, "last_quartile_mean": 4.1923310618585674e-05, "mean": 2.8222844818515114e-05, "normalized": [ 1.092738547299169, 0.5627549631144219, 0.24809914749976855, 0.3821239330736069, 0.2717732575074452, 0.2708922129031054, 0.29822547774823177, 0.4862679168736732, 0.544040724965591, 0.43215299071277247, 0.4170763968939518, 0.42015115877685144, 0.47871065526968887, 0.46826286484802104, 0.4900402959326792, 0.5070735564042542, 0.4633460623422733, 0.5693312811282049, 0.5194930950029935, 0.7472397737749802, 2.399371006579266, 2.8986292719666995, 2.739777767666698, 2.408916709158605, 2.333408286273547, 2.300446003604439, 2.062640270367703, 1.6162771645238014, 1.0435838635901262, 0.9412033869198291, 0.8018738201820883, 0.7840781370955142 ], "peak_layer": 22, "peak_normalized": 2.8986292719666995, "population_cv": 0.8223755795225939, "rest_mean": 2.0090376640129965e-05, "spike_contrast": 3.5906829560228823, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 7.213817298179492e-05, "top_five_layers": [ 22, 23, 24, 21, 25 ], "top_five_spike_overlap": 5 }, "values": [ 3.0840190447634086e-05, 1.5882545994827524e-05, 7.0020637394918595e-06, 1.0784624464577064e-05, 7.670214472454973e-06, 7.6453488873085e-06, 8.416771379415877e-06, 1.3723863958148286e-05, 1.5354376955656335e-05, 1.219658679474378e-05, 1.1771082427003421e-05, 1.1857860954478383e-05, 1.3510576536646113e-05, 1.3215710168879014e-05, 1.3830331226927228e-05, 1.4311058293969836e-05, 1.3076944014756009e-05, 1.606814839760773e-05, 1.4661573004559614e-05, 2.1089232177473605e-05, 6.771707558073103e-05, 8.18075641291216e-05, 7.732432277407497e-05, 6.79864824633114e-05, 6.58554199617356e-05, 6.492513057310134e-05, 5.821357626700774e-05, 4.561593959806487e-05, 2.9452905437210575e-05, 2.6563437131699175e-05, 2.263116039102897e-05, 2.2128915588837117e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 8.982105043742195e-06, "first_to_last_ratio": 0.3255748540372994, "imbalance_abs_log_ratio": 1.122162877451286, "last_quartile_mean": 2.758844834716001e-05, "mean": 1.9313689406885715e-05, "normalized": [ 1.153292392223287, 0.5797609618245384, 0.23076786066825913, 0.37581241874572735, 0.2819306093436648, 0.2851688939012761, 0.30549556090944335, 0.5082847079850745, 0.583859969769007, 0.45907810601554777, 0.45116352890905354, 0.46235698273638653, 0.522619204122844, 0.5090842134923546, 0.5336639455934807, 0.5440001607440246, 0.5095787126873097, 0.6267236930541729, 0.5514908746704607, 0.7356132069066491, 2.430614280460028, 2.906404914690865, 2.7136484598097654, 2.3120655741144565, 2.2482332201724016, 2.1318721582988647, 1.9232163840966459, 1.5686764250532175, 1.0367548344938609, 0.9157520446280638, 0.8238220024989565, 0.7791936973803123 ], "peak_layer": 22, "peak_normalized": 2.906404914690865, "population_cv": 0.7900897792080465, "rest_mean": 1.3869398959219257e-05, "spike_contrast": 3.5122544219484153, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.8712857824284585e-05, "top_five_layers": [ 22, 23, 21, 24, 25 ], "top_five_spike_overlap": 5 }, "values": [ 2.2274331058724783e-05, 1.119732314691646e-05, 4.456978786038235e-06, 7.258324330905452e-06, 5.4451202231575735e-06, 5.5076634453143924e-06, 5.900246378587326e-06, 9.816852980293334e-06, 1.1276490113232285e-05, 8.866491953085642e-06, 8.713632269063964e-06, 8.92981915967539e-06, 1.0093704986502416e-05, 9.832294381340034e-06, 1.0307019692845643e-05, 1.0506650141905993e-05, 9.841844985203352e-06, 1.2104346751584671e-05, 1.0651323464117013e-05, 1.4207405001798179e-05, 4.694412928074598e-05, 5.613340181298554e-05, 5.2410563512239605e-05, 4.4654516386799514e-05, 4.3421678128652275e-05, 4.117431672057137e-05, 3.714440390467644e-05, 3.0296929253381677e-05, 2.0023560864501633e-05, 1.768655056366697e-05, 1.5911042282823473e-05, 1.5049105059006251e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 3.531509560161794e-05, "first_to_last_ratio": 0.2551422661109217, "imbalance_abs_log_ratio": 1.36593398307824, "last_quartile_mean": 0.00013841334930475568, "mean": 8.573219730578785e-05, "normalized": [ 0.9822788357613834, 0.5147377658277125, 0.24077984042300446, 0.3633612864660074, 0.2505573845679934, 0.23871348492801034, 0.2720502738875204, 0.4329080853308244, 0.4899097166580031, 0.3804980486342757, 0.36593190899850525, 0.3643305688887499, 0.412819132135824, 0.40296081445560644, 0.4305724210670739, 0.4502160271371867, 0.3962072388182212, 0.4765863055500514, 0.454434200643566, 0.7197132038245179, 2.2718289409811727, 2.8618271883484865, 2.7794936015049534, 2.53140346971279, 2.4848609397336863, 2.575834812444881, 2.3585524326140215, 1.8181964500308416, 1.0732112926751614, 0.9812549829334379, 0.8189418899610229, 0.8050274550555072 ], "peak_layer": 22, "peak_normalized": 2.8618271883484865, "population_cv": 0.8817284503784036, "rest_mean": 6.055419369119961e-05, "spike_contrast": 3.6610745401896616, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00022169341682456433, "top_five_layers": [ 22, 23, 26, 24, 25 ], "top_five_spike_overlap": 4 }, "values": [ 8.42129229567945e-05, 4.4129599700681865e-05, 2.064258478640113e-05, 3.115176150458865e-05, 2.1480835130205378e-05, 2.0465431589400396e-05, 2.3323467758018523e-05, 3.7114161386853084e-05, 4.200103649054654e-05, 3.262093377998099e-05, 3.137214662274346e-05, 3.123486021650024e-05, 3.539189128787257e-05, 3.454671605140902e-05, 3.691391975735314e-05, 3.859800926875323e-05, 3.3967717172345147e-05, 4.08587911806535e-05, 3.895964255207218e-05, 6.170259439386427e-05, 0.00019476888701319695, 0.00024535073316656053, 0.00023829209385439754, 0.0002170227817259729, 0.0002130325883626938, 0.00022083197836764157, 0.0002022038825089112, 0.00015587797679472715, 9.200876229442656e-05, 8.412514580413699e-05, 7.020968769211322e-05, 6.9016772613395e-05 ] } } } } }, "parent_learned_round06_exact": true, "step": 2000 }, { "local_matrix": null, "parent_learned": { "bits_per_byte": 1.9433676664104056, "forward": { "activation_sha256": { "attention_branch_output": "72af7516114c44d094899f6454538e045d6b49b293ae6fbf480fec93eec73e51", "mlp_branch_output": "16b9daa70ef1c39a83a37551a158c8705d864e7314a8382a1f918981811300d1", "post_attention_state": "90f1dd0e3ce75c2ca97d856d889fb391792298af07000622c435ff53f6d398a7", "post_mlp_state": "89a6b034a7ebb02fe6279d25c4264a8a3e749e8c9a49e8aa75fc34af7ffbb3bf", "pre_attention_input": "655dba72ac4e6dee3e5a00c08f2fe22c99d59ee39e0a5d692a12f5d8d0e85bda", "pre_mlp_input": "bd772114cfba64f2a56e1ed6a7046489b9ab73852568f07a46a3ac39fd02203c" }, "logits_sha256": "000b42d2e473ee11592a1ced2fe607cfcfbbe21b9cb8637453b9d2bc3414fcfe", "loss_nats": 1.347039818763733, "mixer_summary_sha256": "7935548f1014624ed4c9fa535e6d7a55432ce9be32776c3985f675572d6d61e8" }, "loss_nats": 1.347039818763733, "loss_scale": 1.0, "mixers": [ { "branch": "attention", "entropy_mean": -0.0, "group": 1, "latest_source_mass": 1.0, "layer": 1, "max_source_mass": 1.0, "mixer_index": 0, "normalized_entropy": 1.0, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 1.0, "median": 1.0, "p05": 1.0, "p95": 1.0 } ], "sources": 1 }, { "branch": "mlp", "entropy_mean": 0.24809812009334564, "group": 1, "latest_source_mass": 0.07119978964328766, "layer": 1, "max_source_mass": 0.9288002252578735, "mixer_index": 1, "normalized_entropy": 0.35792992751254427, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.9288002252578735, "median": 0.9386287331581116, "p05": 0.858244001865387, "p95": 0.9685006141662598 }, { "label": "current_group_partial", "mean": 0.07119978964328766, "median": 0.06137128174304962, "p05": 0.031499382108449936, "p95": 0.14175599813461304 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6067080497741699, "group": 1, "latest_source_mass": 0.43957531452178955, "layer": 2, "max_source_mass": 0.5604246854782104, "mixer_index": 2, "normalized_entropy": 0.8752946946766094, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.5604246854782104, "median": 0.5413482189178467, "p05": 0.26798251271247864, "p95": 0.8787311315536499 }, { "label": "current_group_partial", "mean": 0.43957531452178955, "median": 0.4586517810821533, "p05": 0.1212688684463501, "p95": 0.7320175170898438 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.3847596049308777, "group": 1, "latest_source_mass": 0.13518528640270233, "layer": 2, "max_source_mass": 0.8648147583007812, "mixer_index": 3, "normalized_entropy": 0.555090773968174, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.8648147583007812, "median": 0.8810019493103027, "p05": 0.7563575506210327, "p95": 0.9309777617454529 }, { "label": "current_group_partial", "mean": 0.13518528640270233, "median": 0.11899800598621368, "p05": 0.06902224570512772, "p95": 0.24364244937896729 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6843481659889221, "group": 1, "latest_source_mass": 0.5056107044219971, "layer": 3, "max_source_mass": 0.5056107044219971, "mixer_index": 4, "normalized_entropy": 0.9873057053136751, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.49438929557800293, "median": 0.4973145127296448, "p05": 0.3897787928581238, "p95": 0.6022077798843384 }, { "label": "current_group_partial", "mean": 0.5056107044219971, "median": 0.502685546875, "p05": 0.3977922201156616, "p95": 0.6102211475372314 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.5728601813316345, "group": 1, "latest_source_mass": 0.30453476309776306, "layer": 3, "max_source_mass": 0.6954652070999146, "mixer_index": 5, "normalized_entropy": 0.8264625427299015, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.6954652070999146, "median": 0.7084143161773682, "p05": 0.453019917011261, "p95": 0.9048184752464294 }, { "label": "current_group_partial", "mean": 0.30453476309776306, "median": 0.29158568382263184, "p05": 0.09518152475357056, "p95": 0.5469801425933838 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.686518132686615, "group": 1, "latest_source_mass": 0.4736926555633545, "layer": 4, "max_source_mass": 0.5263072848320007, "mixer_index": 6, "normalized_entropy": 0.9904363055073309, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.5263072848320007, "median": 0.5291417241096497, "p05": 0.4370424747467041, "p95": 0.6044135093688965 }, { "label": "current_group_partial", "mean": 0.4736926555633545, "median": 0.47085827589035034, "p05": 0.3955864906311035, "p95": 0.5629575252532959 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.5044870376586914, "group": 1, "latest_source_mass": 0.2221563160419464, "layer": 4, "max_source_mass": 0.777843713760376, "mixer_index": 7, "normalized_entropy": 0.7278209474229579, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.777843713760376, "median": 0.7819972634315491, "p05": 0.6222297549247742, "p95": 0.9289263486862183 }, { "label": "current_group_partial", "mean": 0.2221563160419464, "median": 0.21800270676612854, "p05": 0.07107366621494293, "p95": 0.3777702748775482 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6724032163619995, "group": 2, "latest_source_mass": 0.4305936098098755, "layer": 5, "max_source_mass": 0.5694063901901245, "mixer_index": 8, "normalized_entropy": 0.9700727857232454, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.5694063901901245, "median": 0.5728721022605896, "p05": 0.45058131217956543, "p95": 0.6837627291679382 }, { "label": "completed_group_1", "mean": 0.4305936098098755, "median": 0.4271278977394104, "p05": 0.31623730063438416, "p95": 0.5494186878204346 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.9718273282051086, "group": 2, "latest_source_mass": 0.23954525589942932, "layer": 5, "max_source_mass": 0.5544795989990234, "mixer_index": 9, "normalized_entropy": 0.8845953556402437, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.5544795989990234, "median": 0.5475102663040161, "p05": 0.40941980481147766, "p95": 0.7249865531921387 }, { "label": "completed_group_1", "mean": 0.20597517490386963, "median": 0.20141024887561798, "p05": 0.11906205117702484, "p95": 0.3046763241291046 }, { "label": "current_group_partial", "mean": 0.23954525589942932, "median": 0.2397485226392746, "p05": 0.14606745541095734, "p95": 0.33606359362602234 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0856361389160156, "group": 2, "latest_source_mass": 0.36206769943237305, "layer": 6, "max_source_mass": 0.3703838884830475, "mixer_index": 10, "normalized_entropy": 0.9881885994850598, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.3703838884830475, "median": 0.3719092607498169, "p05": 0.31337594985961914, "p95": 0.42367395758628845 }, { "label": "completed_group_1", "mean": 0.26754841208457947, "median": 0.2691181004047394, "p05": 0.24584907293319702, "p95": 0.28693482279777527 }, { "label": "current_group_partial", "mean": 0.36206769943237305, "median": 0.3628925681114197, "p05": 0.31465041637420654, "p95": 0.4040696620941162 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.022249460220337, "group": 2, "latest_source_mass": 0.32249319553375244, "layer": 6, "max_source_mass": 0.46441033482551575, "mixer_index": 11, "normalized_entropy": 0.9304915580906614, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.46441033482551575, "median": 0.47390004992485046, "p05": 0.25647592544555664, "p95": 0.6490365266799927 }, { "label": "completed_group_1", "mean": 0.21309645473957062, "median": 0.2148953229188919, "p05": 0.13959459960460663, "p95": 0.27812671661376953 }, { "label": "current_group_partial", "mean": 0.32249319553375244, "median": 0.31052517890930176, "p05": 0.20389017462730408, "p95": 0.47258833050727844 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0742855072021484, "group": 2, "latest_source_mass": 0.36498188972473145, "layer": 7, "max_source_mass": 0.38770636916160583, "mixer_index": 12, "normalized_entropy": 0.9778568092521033, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.38770636916160583, "median": 0.4003474712371826, "p05": 0.2879825532436371, "p95": 0.45421719551086426 }, { "label": "completed_group_1", "mean": 0.24731171131134033, "median": 0.24118143320083618, "p05": 0.21154659986495972, "p95": 0.2994430363178253 }, { "label": "current_group_partial", "mean": 0.36498188972473145, "median": 0.35881954431533813, "p05": 0.31933724880218506, "p95": 0.42228373885154724 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0207322835922241, "group": 2, "latest_source_mass": 0.4325985312461853, "layer": 7, "max_source_mass": 0.4325985312461853, "mixer_index": 13, "normalized_entropy": 0.9291105644100317, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.3404029607772827, "median": 0.2972515821456909, "p05": 0.1890696883201599, "p95": 0.675995409488678 }, { "label": "completed_group_1", "mean": 0.22699852287769318, "median": 0.2352103292942047, "p05": 0.11332616209983826, "p95": 0.2928347587585449 }, { "label": "current_group_partial", "mean": 0.4325985312461853, "median": 0.4621005654335022, "p05": 0.2054932415485382, "p95": 0.5461148023605347 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0628657341003418, "group": 2, "latest_source_mass": 0.361979603767395, "layer": 8, "max_source_mass": 0.41766518354415894, "mixer_index": 14, "normalized_entropy": 0.9674620838156608, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.41766518354415894, "median": 0.4145643711090088, "p05": 0.36852169036865234, "p95": 0.47784724831581116 }, { "label": "completed_group_1", "mean": 0.22035519778728485, "median": 0.2179521918296814, "p05": 0.191657155752182, "p95": 0.25349321961402893 }, { "label": "current_group_partial", "mean": 0.361979603767395, "median": 0.363964319229126, "p05": 0.3155593276023865, "p95": 0.39730241894721985 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.029329538345337, "group": 2, "latest_source_mass": 0.3858758807182312, "layer": 8, "max_source_mass": 0.42153143882751465, "mixer_index": 15, "normalized_entropy": 0.936936122927619, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.42153143882751465, "median": 0.41038113832473755, "p05": 0.29627615213394165, "p95": 0.6068807244300842 }, { "label": "completed_group_1", "mean": 0.19259268045425415, "median": 0.18828678131103516, "p05": 0.12331291288137436, "p95": 0.2707396447658539 }, { "label": "current_group_partial", "mean": 0.3858758807182312, "median": 0.39191579818725586, "p05": 0.2700805068016052, "p95": 0.4676054120063782 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0849454402923584, "group": 3, "latest_source_mass": 0.3364688754081726, "layer": 9, "max_source_mass": 0.37970075011253357, "mixer_index": 16, "normalized_entropy": 0.9875598985040298, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.37970075011253357, "median": 0.382883220911026, "p05": 0.29113224148750305, "p95": 0.46186232566833496 }, { "label": "completed_group_1", "mean": 0.2838304042816162, "median": 0.282575786113739, "p05": 0.25098085403442383, "p95": 0.31989675760269165 }, { "label": "completed_group_2", "mean": 0.3364688754081726, "median": 0.33596307039260864, "p05": 0.28113189339637756, "p95": 0.3942520022392273 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.3144235610961914, "group": 3, "latest_source_mass": 0.30927807092666626, "layer": 9, "max_source_mass": 0.3205832242965698, "mixer_index": 17, "normalized_entropy": 0.9481561766105434, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.3205832242965698, "median": 0.3228584825992584, "p05": 0.20318874716758728, "p95": 0.49334216117858887 }, { "label": "completed_group_1", "mean": 0.13048580288887024, "median": 0.1293262243270874, "p05": 0.08803658187389374, "p95": 0.17583128809928894 }, { "label": "completed_group_2", "mean": 0.2396528720855713, "median": 0.23850613832473755, "p05": 0.15605276823043823, "p95": 0.3097888231277466 }, { "label": "current_group_partial", "mean": 0.30927807092666626, "median": 0.3110916018486023, "p05": 0.24646314978599548, "p95": 0.36450129747390747 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3647149801254272, "group": 3, "latest_source_mass": 0.24869562685489655, "layer": 10, "max_source_mass": 0.3188267946243286, "mixer_index": 18, "normalized_entropy": 0.9844337670269171, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.3188267946243286, "median": 0.32193225622177124, "p05": 0.2530626356601715, "p95": 0.38645845651626587 }, { "label": "completed_group_1", "mean": 0.2379944920539856, "median": 0.23598769307136536, "p05": 0.21630848944187164, "p95": 0.2653774321079254 }, { "label": "completed_group_2", "mean": 0.19448311626911163, "median": 0.19229988753795624, "p05": 0.16299879550933838, "p95": 0.23252975940704346 }, { "label": "current_group_partial", "mean": 0.24869562685489655, "median": 0.24343957006931305, "p05": 0.21172824501991272, "p95": 0.29676955938339233 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.2778496742248535, "group": 3, "latest_source_mass": 0.2590479850769043, "layer": 10, "max_source_mass": 0.37896889448165894, "mixer_index": 19, "normalized_entropy": 0.9217736940028869, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.37896889448165894, "median": 0.3914150297641754, "p05": 0.14146459102630615, "p95": 0.6288160681724548 }, { "label": "completed_group_1", "mean": 0.16353653371334076, "median": 0.162285715341568, "p05": 0.08837783336639404, "p95": 0.23340767621994019 }, { "label": "completed_group_2", "mean": 0.1984466314315796, "median": 0.19906753301620483, "p05": 0.09471487253904343, "p95": 0.29035234451293945 }, { "label": "current_group_partial", "mean": 0.2590479850769043, "median": 0.2469818890094757, "p05": 0.1562758982181549, "p95": 0.3916419744491577 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3567583560943604, "group": 3, "latest_source_mass": 0.3088001608848572, "layer": 11, "max_source_mass": 0.3088001608848572, "mixer_index": 20, "normalized_entropy": 0.978694276010998, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.29770222306251526, "median": 0.29406338930130005, "p05": 0.24821004271507263, "p95": 0.3566911518573761 }, { "label": "completed_group_1", "mean": 0.21492168307304382, "median": 0.21014970541000366, "p05": 0.18107125163078308, "p95": 0.26316314935684204 }, { "label": "completed_group_2", "mean": 0.17857596278190613, "median": 0.17864474654197693, "p05": 0.15527817606925964, "p95": 0.2019544541835785 }, { "label": "current_group_partial", "mean": 0.3088001608848572, "median": 0.309349000453949, "p05": 0.2659364938735962, "p95": 0.3488244414329529 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.2773678302764893, "group": 3, "latest_source_mass": 0.24117937684059143, "layer": 11, "max_source_mass": 0.3856483995914459, "mixer_index": 21, "normalized_entropy": 0.9214261170654932, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.3856483995914459, "median": 0.4236947298049927, "p05": 0.12469026446342468, "p95": 0.588103175163269 }, { "label": "completed_group_1", "mean": 0.18032220005989075, "median": 0.17527258396148682, "p05": 0.10318915545940399, "p95": 0.2606950104236603 }, { "label": "completed_group_2", "mean": 0.1928499937057495, "median": 0.18568618595600128, "p05": 0.12013933062553406, "p95": 0.28692901134490967 }, { "label": "current_group_partial", "mean": 0.24117937684059143, "median": 0.2192210555076599, "p05": 0.14662782847881317, "p95": 0.4063720405101776 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3655540943145752, "group": 3, "latest_source_mass": 0.29187142848968506, "layer": 12, "max_source_mass": 0.29819542169570923, "mixer_index": 22, "normalized_entropy": 0.9850390599666288, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.29819542169570923, "median": 0.29867103695869446, "p05": 0.2453947365283966, "p95": 0.34859126806259155 }, { "label": "completed_group_1", "mean": 0.20063453912734985, "median": 0.19907723367214203, "p05": 0.16811034083366394, "p95": 0.23844538629055023 }, { "label": "completed_group_2", "mean": 0.20929867029190063, "median": 0.21011404693126678, "p05": 0.18459835648536682, "p95": 0.23381994664669037 }, { "label": "current_group_partial", "mean": 0.29187142848968506, "median": 0.29417213797569275, "p05": 0.24766536056995392, "p95": 0.32690057158470154 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.2754871845245361, "group": 3, "latest_source_mass": 0.22957558929920197, "layer": 12, "max_source_mass": 0.42651012539863586, "mixer_index": 23, "normalized_entropy": 0.9200695179154873, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.42651012539863586, "median": 0.44201791286468506, "p05": 0.24115881323814392, "p95": 0.5910159945487976 }, { "label": "completed_group_1", "mean": 0.1713070571422577, "median": 0.170497864484787, "p05": 0.12993231415748596, "p95": 0.21631143987178802 }, { "label": "completed_group_2", "mean": 0.1726072132587433, "median": 0.16530022025108337, "p05": 0.11561842262744904, "p95": 0.2528795301914215 }, { "label": "current_group_partial", "mean": 0.22957558929920197, "median": 0.2182570993900299, "p05": 0.14037305116653442, "p95": 0.3425685167312622 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3596348762512207, "group": 4, "latest_source_mass": 0.3104133605957031, "layer": 13, "max_source_mass": 0.3104133605957031, "mixer_index": 24, "normalized_entropy": 0.9807692466936578, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.2750997543334961, "median": 0.2747533321380615, "p05": 0.2329672873020172, "p95": 0.31678152084350586 }, { "label": "completed_group_1", "mean": 0.22746169567108154, "median": 0.215274840593338, "p05": 0.18571145832538605, "p95": 0.3119947016239166 }, { "label": "completed_group_2", "mean": 0.18702521920204163, "median": 0.1861300766468048, "p05": 0.16768434643745422, "p95": 0.20858913660049438 }, { "label": "completed_group_3", "mean": 0.3104133605957031, "median": 0.32007908821105957, "p05": 0.2161477506160736, "p95": 0.3817093074321747 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.5752925872802734, "group": 4, "latest_source_mass": 0.20874352753162384, "layer": 13, "max_source_mass": 0.28211504220962524, "mixer_index": 25, "normalized_entropy": 0.9787843166300303, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.28211504220962524, "median": 0.2998196482658386, "p05": 0.19197332859039307, "p95": 0.3606031835079193 }, { "label": "completed_group_1", "mean": 0.1675817370414734, "median": 0.16741523146629333, "p05": 0.14403808116912842, "p95": 0.19205224514007568 }, { "label": "completed_group_2", "mean": 0.1759152114391327, "median": 0.17603915929794312, "p05": 0.138168603181839, "p95": 0.21206450462341309 }, { "label": "completed_group_3", "mean": 0.16564446687698364, "median": 0.16148409247398376, "p05": 0.12781348824501038, "p95": 0.21234022080898285 }, { "label": "current_group_partial", "mean": 0.20874352753162384, "median": 0.2069983333349228, "p05": 0.17109251022338867, "p95": 0.24858909845352173 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.5704567432403564, "group": 4, "latest_source_mass": 0.29884350299835205, "layer": 14, "max_source_mass": 0.29884350299835205, "mixer_index": 26, "normalized_entropy": 0.975779637789948, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.21000900864601135, "median": 0.21329987049102783, "p05": 0.17762595415115356, "p95": 0.2425374686717987 }, { "label": "completed_group_1", "mean": 0.18079037964344025, "median": 0.17888139188289642, "p05": 0.15963314473628998, "p95": 0.20616433024406433 }, { "label": "completed_group_2", "mean": 0.13544103503227234, "median": 0.1345825046300888, "p05": 0.12005693465471268, "p95": 0.1533820480108261 }, { "label": "completed_group_3", "mean": 0.1749160885810852, "median": 0.17272356152534485, "p05": 0.15442612767219543, "p95": 0.20184364914894104 }, { "label": "current_group_partial", "mean": 0.29884350299835205, "median": 0.3006042242050171, "p05": 0.2674207091331482, "p95": 0.3227018713951111 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.579707384109497, "group": 4, "latest_source_mass": 0.24509160220623016, "layer": 14, "max_source_mass": 0.24509160220623016, "mixer_index": 27, "normalized_entropy": 0.98152738412901, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.21335941553115845, "median": 0.21055345237255096, "p05": 0.12470410019159317, "p95": 0.33470356464385986 }, { "label": "completed_group_1", "mean": 0.18788450956344604, "median": 0.18773096799850464, "p05": 0.14719606935977936, "p95": 0.22808396816253662 }, { "label": "completed_group_2", "mean": 0.20250459015369415, "median": 0.20350772142410278, "p05": 0.15922458469867706, "p95": 0.24294495582580566 }, { "label": "completed_group_3", "mean": 0.1511598825454712, "median": 0.14913730323314667, "p05": 0.12029191851615906, "p95": 0.1872422844171524 }, { "label": "current_group_partial", "mean": 0.24509160220623016, "median": 0.23680704832077026, "p05": 0.18374761939048767, "p95": 0.3342677652835846 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.5663042068481445, "group": 4, "latest_source_mass": 0.29828590154647827, "layer": 15, "max_source_mass": 0.29828590154647827, "mixer_index": 28, "normalized_entropy": 0.9731995218624366, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.21676908433437347, "median": 0.2142370641231537, "p05": 0.1767536848783493, "p95": 0.2722740173339844 }, { "label": "completed_group_1", "mean": 0.19467836618423462, "median": 0.19520726799964905, "p05": 0.16570945084095, "p95": 0.2210082858800888 }, { "label": "completed_group_2", "mean": 0.13713446259498596, "median": 0.13748224079608917, "p05": 0.12025479972362518, "p95": 0.15284715592861176 }, { "label": "completed_group_3", "mean": 0.15313217043876648, "median": 0.153159961104393, "p05": 0.13474825024604797, "p95": 0.1712232530117035 }, { "label": "current_group_partial", "mean": 0.29828590154647827, "median": 0.29950791597366333, "p05": 0.2589420676231384, "p95": 0.33298033475875854 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.5384478569030762, "group": 4, "latest_source_mass": 0.29070425033569336, "layer": 15, "max_source_mass": 0.29070425033569336, "mixer_index": 29, "normalized_entropy": 0.9558913984922479, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.21904908120632172, "median": 0.20143723487854004, "p05": 0.09117002040147781, "p95": 0.4469224512577057 }, { "label": "completed_group_1", "mean": 0.15864285826683044, "median": 0.15922614932060242, "p05": 0.10943489521741867, "p95": 0.2047940045595169 }, { "label": "completed_group_2", "mean": 0.18178996443748474, "median": 0.18768036365509033, "p05": 0.10846641659736633, "p95": 0.23086166381835938 }, { "label": "completed_group_3", "mean": 0.14981383085250854, "median": 0.14754682779312134, "p05": 0.10115746408700943, "p95": 0.19903366267681122 }, { "label": "current_group_partial", "mean": 0.29070425033569336, "median": 0.2898399233818054, "p05": 0.20691633224487305, "p95": 0.37558960914611816 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.552668571472168, "group": 4, "latest_source_mass": 0.30334532260894775, "layer": 16, "max_source_mass": 0.30334532260894775, "mixer_index": 30, "normalized_entropy": 0.9647272252484255, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.22710487246513367, "median": 0.2245989590883255, "p05": 0.16175971925258636, "p95": 0.2988150715827942 }, { "label": "completed_group_1", "mean": 0.1922147125005722, "median": 0.1916719675064087, "p05": 0.15179888904094696, "p95": 0.23553937673568726 }, { "label": "completed_group_2", "mean": 0.13047492504119873, "median": 0.12986470758914948, "p05": 0.10791919380426407, "p95": 0.15449906885623932 }, { "label": "completed_group_3", "mean": 0.14686015248298645, "median": 0.14501844346523285, "p05": 0.12235601246356964, "p95": 0.17693395912647247 }, { "label": "current_group_partial", "mean": 0.30334532260894775, "median": 0.29679566621780396, "p05": 0.24902349710464478, "p95": 0.3774981200695038 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.5210195779800415, "group": 4, "latest_source_mass": 0.282525897026062, "layer": 16, "max_source_mass": 0.282525897026062, "mixer_index": 31, "normalized_entropy": 0.9450625999481175, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.2812899947166443, "median": 0.2745441794395447, "p05": 0.15230709314346313, "p95": 0.4724813997745514 }, { "label": "completed_group_1", "mean": 0.14511607587337494, "median": 0.14084157347679138, "p05": 0.10224629938602448, "p95": 0.2002197802066803 }, { "label": "completed_group_2", "mean": 0.15909329056739807, "median": 0.15865719318389893, "p05": 0.10898765176534653, "p95": 0.21094731986522675 }, { "label": "completed_group_3", "mean": 0.1319747418165207, "median": 0.1301102191209793, "p05": 0.08913854509592056, "p95": 0.17862913012504578 }, { "label": "current_group_partial", "mean": 0.282525897026062, "median": 0.28236913681030273, "p05": 0.19806379079818726, "p95": 0.36864838004112244 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.5376527309417725, "group": 5, "latest_source_mass": 0.287123441696167, "layer": 17, "max_source_mass": 0.287123441696167, "mixer_index": 32, "normalized_entropy": 0.9553973589551147, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.23905543982982635, "median": 0.23701313138008118, "p05": 0.18620719015598297, "p95": 0.2925828695297241 }, { "label": "completed_group_1", "mean": 0.21673071384429932, "median": 0.2044925093650818, "p05": 0.15778198838233948, "p95": 0.321443647146225 }, { "label": "completed_group_2", "mean": 0.0950920432806015, "median": 0.09378060698509216, "p05": 0.07889896631240845, "p95": 0.11442621052265167 }, { "label": "completed_group_3", "mean": 0.16199833154678345, "median": 0.16117054224014282, "p05": 0.12811943888664246, "p95": 0.19840528070926666 }, { "label": "completed_group_4", "mean": 0.287123441696167, "median": 0.29111331701278687, "p05": 0.2107674777507782, "p95": 0.34516024589538574 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.7209875583648682, "group": 5, "latest_source_mass": 0.1365262269973755, "layer": 17, "max_source_mass": 0.25328749418258667, "mixer_index": 33, "normalized_entropy": 0.9605014444859178, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1703755259513855, "median": 0.1390259861946106, "p05": 0.08010496944189072, "p95": 0.3672885000705719 }, { "label": "completed_group_1", "mean": 0.14151468873023987, "median": 0.1361694037914276, "p05": 0.10555511713027954, "p95": 0.19257718324661255 }, { "label": "completed_group_2", "mean": 0.17087554931640625, "median": 0.17344608902931213, "p05": 0.10519347339868546, "p95": 0.22031483054161072 }, { "label": "completed_group_3", "mean": 0.12742051482200623, "median": 0.12381769716739655, "p05": 0.0888485461473465, "p95": 0.17263755202293396 }, { "label": "completed_group_4", "mean": 0.25328749418258667, "median": 0.25078505277633667, "p05": 0.163196861743927, "p95": 0.35207411646842957 }, { "label": "current_group_partial", "mean": 0.1365262269973755, "median": 0.13269345462322235, "p05": 0.0852486714720726, "p95": 0.20064347982406616 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.7200093269348145, "group": 5, "latest_source_mass": 0.27521812915802, "layer": 18, "max_source_mass": 0.27521812915802, "mixer_index": 34, "normalized_entropy": 0.9599554831295783, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.18992580473423004, "median": 0.19383516907691956, "p05": 0.1371862292289734, "p95": 0.23474527895450592 }, { "label": "completed_group_1", "mean": 0.16036251187324524, "median": 0.1590055525302887, "p05": 0.14324960112571716, "p95": 0.1809857338666916 }, { "label": "completed_group_2", "mean": 0.08439569175243378, "median": 0.08315694332122803, "p05": 0.07186377048492432, "p95": 0.10051116347312927 }, { "label": "completed_group_3", "mean": 0.11250905692577362, "median": 0.11015664041042328, "p05": 0.09692119807004929, "p95": 0.13486211001873016 }, { "label": "completed_group_4", "mean": 0.1775888353586197, "median": 0.1758042573928833, "p05": 0.1488552987575531, "p95": 0.21211811900138855 }, { "label": "current_group_partial", "mean": 0.27521812915802, "median": 0.2752543091773987, "p05": 0.25022760033607483, "p95": 0.3000151515007019 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.753357172012329, "group": 5, "latest_source_mass": 0.24494445323944092, "layer": 18, "max_source_mass": 0.24494445323944092, "mixer_index": 35, "normalized_entropy": 0.978567269839924, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.15539543330669403, "median": 0.14567650854587555, "p05": 0.1150466799736023, "p95": 0.23550888895988464 }, { "label": "completed_group_1", "mean": 0.1388990879058838, "median": 0.1368958055973053, "p05": 0.11385980248451233, "p95": 0.17140795290470123 }, { "label": "completed_group_2", "mean": 0.15745997428894043, "median": 0.15778976678848267, "p05": 0.13043276965618134, "p95": 0.1834889054298401 }, { "label": "completed_group_3", "mean": 0.12427005916833878, "median": 0.12222100794315338, "p05": 0.10104347765445709, "p95": 0.1512138545513153 }, { "label": "completed_group_4", "mean": 0.17903099954128265, "median": 0.1781059056520462, "p05": 0.14208883047103882, "p95": 0.21939483284950256 }, { "label": "current_group_partial", "mean": 0.24494445323944092, "median": 0.24745827913284302, "p05": 0.16898170113563538, "p95": 0.3117744028568268 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.7199265956878662, "group": 5, "latest_source_mass": 0.2730581760406494, "layer": 19, "max_source_mass": 0.2730581760406494, "mixer_index": 36, "normalized_entropy": 0.9599093099415087, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.19207912683486938, "median": 0.19417056441307068, "p05": 0.14205873012542725, "p95": 0.2466108500957489 }, { "label": "completed_group_1", "mean": 0.17904852330684662, "median": 0.17943383753299713, "p05": 0.15607136487960815, "p95": 0.2006712406873703 }, { "label": "completed_group_2", "mean": 0.08410340547561646, "median": 0.08364531397819519, "p05": 0.07357755303382874, "p95": 0.09571512043476105 }, { "label": "completed_group_3", "mean": 0.1121990978717804, "median": 0.11130240559577942, "p05": 0.0962604433298111, "p95": 0.13100717961788177 }, { "label": "completed_group_4", "mean": 0.15951167047023773, "median": 0.1591540426015854, "p05": 0.13084900379180908, "p95": 0.1905163675546646 }, { "label": "current_group_partial", "mean": 0.2730581760406494, "median": 0.27290621399879456, "p05": 0.241956889629364, "p95": 0.3052297830581665 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.6756346225738525, "group": 5, "latest_source_mass": 0.3124367296695709, "layer": 19, "max_source_mass": 0.3124367296695709, "mixer_index": 37, "normalized_entropy": 0.9351894890756556, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.15863081812858582, "median": 0.1342058777809143, "p05": 0.07548131048679352, "p95": 0.3739146590232849 }, { "label": "completed_group_1", "mean": 0.11242728680372238, "median": 0.11213450133800507, "p05": 0.08389876782894135, "p95": 0.14075449109077454 }, { "label": "completed_group_2", "mean": 0.15970337390899658, "median": 0.16262288391590118, "p05": 0.10122501850128174, "p95": 0.2050495743751526 }, { "label": "completed_group_3", "mean": 0.09774985909461975, "median": 0.09510388225317001, "p05": 0.06857988238334656, "p95": 0.12727102637290955 }, { "label": "completed_group_4", "mean": 0.15905189514160156, "median": 0.1553158164024353, "p05": 0.11221202462911606, "p95": 0.2187761515378952 }, { "label": "current_group_partial", "mean": 0.3124367296695709, "median": 0.3196638822555542, "p05": 0.18551334738731384, "p95": 0.4139381945133209 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.7271867990493774, "group": 5, "latest_source_mass": 0.2614278793334961, "layer": 20, "max_source_mass": 0.2614278793334961, "mixer_index": 38, "normalized_entropy": 0.9639613065884912, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.19560003280639648, "median": 0.1968384087085724, "p05": 0.1501731276512146, "p95": 0.2440420687198639 }, { "label": "completed_group_1", "mean": 0.18333327770233154, "median": 0.18228237330913544, "p05": 0.1655699908733368, "p95": 0.2043052464723587 }, { "label": "completed_group_2", "mean": 0.08475088328123093, "median": 0.08410285413265228, "p05": 0.07304102182388306, "p95": 0.09892263263463974 }, { "label": "completed_group_3", "mean": 0.11598999798297882, "median": 0.1141754686832428, "p05": 0.10139824450016022, "p95": 0.1362377256155014 }, { "label": "completed_group_4", "mean": 0.15889792144298553, "median": 0.15817126631736755, "p05": 0.13111311197280884, "p95": 0.18958830833435059 }, { "label": "current_group_partial", "mean": 0.2614278793334961, "median": 0.2622074484825134, "p05": 0.2309473156929016, "p95": 0.2893397808074951 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.6916961669921875, "group": 5, "latest_source_mass": 0.3091394901275635, "layer": 20, "max_source_mass": 0.3091394901275635, "mixer_index": 39, "normalized_entropy": 0.9441536076943532, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.19065655767917633, "median": 0.1910099983215332, "p05": 0.09996911883354187, "p95": 0.313973069190979 }, { "label": "completed_group_1", "mean": 0.12983909249305725, "median": 0.12771058082580566, "p05": 0.10737630724906921, "p95": 0.15882958471775055 }, { "label": "completed_group_2", "mean": 0.11857748031616211, "median": 0.11902420967817307, "p05": 0.08811743557453156, "p95": 0.14735448360443115 }, { "label": "completed_group_3", "mean": 0.09883742034435272, "median": 0.0951075479388237, "p05": 0.07493072748184204, "p95": 0.1301351934671402 }, { "label": "completed_group_4", "mean": 0.1529499888420105, "median": 0.15091747045516968, "p05": 0.11475001275539398, "p95": 0.19942505657672882 }, { "label": "current_group_partial", "mean": 0.3091394901275635, "median": 0.3146688640117645, "p05": 0.2252732366323471, "p95": 0.3756124973297119 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.7504222393035889, "group": 6, "latest_source_mass": 0.2152354121208191, "layer": 21, "max_source_mass": 0.2152354121208191, "mixer_index": 40, "normalized_entropy": 0.9769292527069633, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.20809361338615417, "median": 0.21471115946769714, "p05": 0.13097207248210907, "p95": 0.2735797166824341 }, { "label": "completed_group_1", "mean": 0.19103102385997772, "median": 0.19057458639144897, "p05": 0.16917173564434052, "p95": 0.2132101207971573 }, { "label": "completed_group_2", "mean": 0.10743936896324158, "median": 0.10638970136642456, "p05": 0.09155704826116562, "p95": 0.12698283791542053 }, { "label": "completed_group_3", "mean": 0.1194620281457901, "median": 0.11578401178121567, "p05": 0.10080886632204056, "p95": 0.14768129587173462 }, { "label": "completed_group_4", "mean": 0.15873853862285614, "median": 0.15696901082992554, "p05": 0.13252565264701843, "p95": 0.19312383234500885 }, { "label": "completed_group_5", "mean": 0.2152354121208191, "median": 0.21616290509700775, "p05": 0.18420100212097168, "p95": 0.24399688839912415 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.5379457473754883, "group": 6, "latest_source_mass": 0.515034556388855, "layer": 21, "max_source_mass": 0.515034556388855, "mixer_index": 41, "normalized_entropy": 0.7903477702308708, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.09571269154548645, "median": 0.08428268879652023, "p05": 0.04900683835148811, "p95": 0.1901935338973999 }, { "label": "completed_group_1", "mean": 0.07078731805086136, "median": 0.06967149674892426, "p05": 0.05498572438955307, "p95": 0.08918046206235886 }, { "label": "completed_group_2", "mean": 0.0824701264500618, "median": 0.08152244985103607, "p05": 0.06332080066204071, "p95": 0.10414593666791916 }, { "label": "completed_group_3", "mean": 0.060012347996234894, "median": 0.058421529829502106, "p05": 0.04666804522275925, "p95": 0.07636430114507675 }, { "label": "completed_group_4", "mean": 0.08145605027675629, "median": 0.0806138813495636, "p05": 0.061518602073192596, "p95": 0.10454171150922775 }, { "label": "completed_group_5", "mean": 0.09452696144580841, "median": 0.09397821128368378, "p05": 0.06939592957496643, "p95": 0.12165899574756622 }, { "label": "current_group_partial", "mean": 0.515034556388855, "median": 0.5195302963256836, "p05": 0.41491252183914185, "p95": 0.5988897681236267 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.7319592237472534, "group": 6, "latest_source_mass": 0.3903183937072754, "layer": 22, "max_source_mass": 0.3903183937072754, "mixer_index": 42, "normalized_entropy": 0.8900509741357138, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.13678354024887085, "median": 0.14219121634960175, "p05": 0.0800691545009613, "p95": 0.18056681752204895 }, { "label": "completed_group_1", "mean": 0.11740624904632568, "median": 0.11779546737670898, "p05": 0.09323577582836151, "p95": 0.13856223225593567 }, { "label": "completed_group_2", "mean": 0.06668868660926819, "median": 0.06673324108123779, "p05": 0.05682401731610298, "p95": 0.07633707672357559 }, { "label": "completed_group_3", "mean": 0.07360264658927917, "median": 0.07287117838859558, "p05": 0.06238633394241333, "p95": 0.0858106017112732 }, { "label": "completed_group_4", "mean": 0.09287553280591965, "median": 0.0919380784034729, "p05": 0.07644987106323242, "p95": 0.11259186267852783 }, { "label": "completed_group_5", "mean": 0.12232495099306107, "median": 0.12153243273496628, "p05": 0.10421743988990784, "p95": 0.14276890456676483 }, { "label": "current_group_partial", "mean": 0.3903183937072754, "median": 0.382642537355423, "p05": 0.32760486006736755, "p95": 0.4824257791042328 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.4663677215576172, "group": 6, "latest_source_mass": 0.5484944581985474, "layer": 22, "max_source_mass": 0.5484944581985474, "mixer_index": 43, "normalized_entropy": 0.7535639414129677, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.09485072642564774, "median": 0.07708277553319931, "p05": 0.041287682950496674, "p95": 0.2252470850944519 }, { "label": "completed_group_1", "mean": 0.07167200744152069, "median": 0.07091978192329407, "p05": 0.05350632965564728, "p95": 0.09384822845458984 }, { "label": "completed_group_2", "mean": 0.0841417908668518, "median": 0.08349095284938812, "p05": 0.060824211686849594, "p95": 0.11007672548294067 }, { "label": "completed_group_3", "mean": 0.05784624069929123, "median": 0.056044802069664, "p05": 0.04343996196985245, "p95": 0.07539206743240356 }, { "label": "completed_group_4", "mean": 0.07052354514598846, "median": 0.06955709308385849, "p05": 0.048022691160440445, "p95": 0.09658979624509811 }, { "label": "completed_group_5", "mean": 0.0724712610244751, "median": 0.07230593264102936, "p05": 0.05036160722374916, "p95": 0.09567554295063019 }, { "label": "current_group_partial", "mean": 0.5484944581985474, "median": 0.5530485510826111, "p05": 0.4294937252998352, "p95": 0.6537213325500488 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.749481201171875, "group": 6, "latest_source_mass": 0.37420469522476196, "layer": 23, "max_source_mass": 0.37420469522476196, "mixer_index": 44, "normalized_entropy": 0.8990554892892669, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.14530612528324127, "median": 0.1497679054737091, "p05": 0.08296400308609009, "p95": 0.20158618688583374 }, { "label": "completed_group_1", "mean": 0.12230369448661804, "median": 0.12149038910865784, "p05": 0.10498286038637161, "p95": 0.14227813482284546 }, { "label": "completed_group_2", "mean": 0.0660901814699173, "median": 0.06578399986028671, "p05": 0.05611872673034668, "p95": 0.07736622542142868 }, { "label": "completed_group_3", "mean": 0.07630561292171478, "median": 0.07523077726364136, "p05": 0.06339914351701736, "p95": 0.09089048206806183 }, { "label": "completed_group_4", "mean": 0.09453228116035461, "median": 0.09349468350410461, "p05": 0.07686570286750793, "p95": 0.11586424708366394 }, { "label": "completed_group_5", "mean": 0.12125740945339203, "median": 0.12067118287086487, "p05": 0.10250546038150787, "p95": 0.14243635535240173 }, { "label": "current_group_partial", "mean": 0.37420469522476196, "median": 0.36897724866867065, "p05": 0.30640295147895813, "p95": 0.4604528248310089 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.472255825996399, "group": 6, "latest_source_mass": 0.5438836812973022, "layer": 23, "max_source_mass": 0.5438836812973022, "mixer_index": 45, "normalized_entropy": 0.7565898285237576, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.10688216984272003, "median": 0.0909440815448761, "p05": 0.04993019625544548, "p95": 0.22096660733222961 }, { "label": "completed_group_1", "mean": 0.07465942203998566, "median": 0.07390505075454712, "p05": 0.05582176893949509, "p95": 0.09690531343221664 }, { "label": "completed_group_2", "mean": 0.08327433466911316, "median": 0.08253603428602219, "p05": 0.05906296521425247, "p95": 0.10942579060792923 }, { "label": "completed_group_3", "mean": 0.054589346051216125, "median": 0.052618082612752914, "p05": 0.039012178778648376, "p95": 0.07247936725616455 }, { "label": "completed_group_4", "mean": 0.0637175589799881, "median": 0.062460802495479584, "p05": 0.04167243465781212, "p95": 0.09087979793548584 }, { "label": "completed_group_5", "mean": 0.07299347221851349, "median": 0.07232271879911423, "p05": 0.05079510435461998, "p95": 0.09688537567853928 }, { "label": "current_group_partial", "mean": 0.5438836812973022, "median": 0.5471436977386475, "p05": 0.42356419563293457, "p95": 0.6585544347763062 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.7924723625183105, "group": 6, "latest_source_mass": 0.3345995247364044, "layer": 24, "max_source_mass": 0.3345995247364044, "mixer_index": 46, "normalized_entropy": 0.9211485758417507, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.15522946417331696, "median": 0.16076582670211792, "p05": 0.0863914042711258, "p95": 0.2195497751235962 }, { "label": "completed_group_1", "mean": 0.13724806904792786, "median": 0.13687172532081604, "p05": 0.1200050488114357, "p95": 0.1560649424791336 }, { "label": "completed_group_2", "mean": 0.06889298558235168, "median": 0.06847138702869415, "p05": 0.058477431535720825, "p95": 0.08076588809490204 }, { "label": "completed_group_3", "mean": 0.08198929578065872, "median": 0.08025878667831421, "p05": 0.06727518886327744, "p95": 0.10007832199335098 }, { "label": "completed_group_4", "mean": 0.09816981852054596, "median": 0.09709671884775162, "p05": 0.07776231318712234, "p95": 0.12178758531808853 }, { "label": "completed_group_5", "mean": 0.123870849609375, "median": 0.12361469119787216, "p05": 0.1038728728890419, "p95": 0.14525073766708374 }, { "label": "current_group_partial", "mean": 0.3345995247364044, "median": 0.3317973017692566, "p05": 0.265953928232193, "p95": 0.412749707698822 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.5755163431167603, "group": 6, "latest_source_mass": 0.48653730750083923, "layer": 24, "max_source_mass": 0.48653730750083923, "mixer_index": 47, "normalized_entropy": 0.8096552371041545, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.12274101376533508, "median": 0.10600537061691284, "p05": 0.06405961513519287, "p95": 0.25983333587646484 }, { "label": "completed_group_1", "mean": 0.08900541067123413, "median": 0.08780360221862793, "p05": 0.06876012682914734, "p95": 0.11320354044437408 }, { "label": "completed_group_2", "mean": 0.08551642298698425, "median": 0.08451470732688904, "p05": 0.06253526359796524, "p95": 0.1114322617650032 }, { "label": "completed_group_3", "mean": 0.06285787373781204, "median": 0.061114855110645294, "p05": 0.045210886746644974, "p95": 0.08213447034358978 }, { "label": "completed_group_4", "mean": 0.07429315149784088, "median": 0.07252945005893707, "p05": 0.049904488027095795, "p95": 0.10469712316989899 }, { "label": "completed_group_5", "mean": 0.07904879748821259, "median": 0.07821422815322876, "p05": 0.05642634630203247, "p95": 0.10449491441249847 }, { "label": "current_group_partial", "mean": 0.48653730750083923, "median": 0.49373340606689453, "p05": 0.3549031913280487, "p95": 0.5985680222511292 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.8390922546386719, "group": 7, "latest_source_mass": 0.29349833726882935, "layer": 25, "max_source_mass": 0.29349833726882935, "mixer_index": 48, "normalized_entropy": 0.945106461123861, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.15884169936180115, "median": 0.15805873274803162, "p05": 0.09690988063812256, "p95": 0.2504432201385498 }, { "label": "completed_group_1", "mean": 0.14726480841636658, "median": 0.14679506421089172, "p05": 0.12764519453048706, "p95": 0.16903650760650635 }, { "label": "completed_group_2", "mean": 0.08053630590438843, "median": 0.08051620423793793, "p05": 0.06807594001293182, "p95": 0.09329371899366379 }, { "label": "completed_group_3", "mean": 0.0909707322716713, "median": 0.08902134001255035, "p05": 0.07499626278877258, "p95": 0.11060230433940887 }, { "label": "completed_group_4", "mean": 0.10738551616668701, "median": 0.10607586801052094, "p05": 0.08672098070383072, "p95": 0.1325560212135315 }, { "label": "completed_group_5", "mean": 0.12150255590677261, "median": 0.12205702066421509, "p05": 0.0993877649307251, "p95": 0.14230164885520935 }, { "label": "completed_group_6", "mean": 0.29349833726882935, "median": 0.29394760727882385, "p05": 0.22660650312900543, "p95": 0.3548165559768677 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.6550344228744507, "group": 7, "latest_source_mass": 0.3589511513710022, "layer": 25, "max_source_mass": 0.3589511513710022, "mixer_index": 49, "normalized_entropy": 0.795903318127166, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.0844201073050499, "median": 0.0718768984079361, "p05": 0.02839931659400463, "p95": 0.21728233993053436 }, { "label": "completed_group_1", "mean": 0.06695197522640228, "median": 0.06580687314271927, "p05": 0.04618899151682854, "p95": 0.09124172478914261 }, { "label": "completed_group_2", "mean": 0.061310961842536926, "median": 0.06096746027469635, "p05": 0.040395740419626236, "p95": 0.08348730951547623 }, { "label": "completed_group_3", "mean": 0.0447731651365757, "median": 0.042184896767139435, "p05": 0.02882835641503334, "p95": 0.06167382374405861 }, { "label": "completed_group_4", "mean": 0.04748315364122391, "median": 0.04516277089715004, "p05": 0.0278231929987669, "p95": 0.0743657797574997 }, { "label": "completed_group_5", "mean": 0.045562684535980225, "median": 0.044303566217422485, "p05": 0.03103901632130146, "p95": 0.06420586258172989 }, { "label": "completed_group_6", "mean": 0.2905467748641968, "median": 0.2895723581314087, "p05": 0.1524062305688858, "p95": 0.43639296293258667 }, { "label": "current_group_partial", "mean": 0.3589511513710022, "median": 0.3606260418891907, "p05": 0.16963356733322144, "p95": 0.546379804611206 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.8679088354110718, "group": 7, "latest_source_mass": 0.32435470819473267, "layer": 26, "max_source_mass": 0.32435470819473267, "mixer_index": 50, "normalized_entropy": 0.8982742712267442, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.11674155294895172, "median": 0.12052157521247864, "p05": 0.05360257253050804, "p95": 0.17403334379196167 }, { "label": "completed_group_1", "mean": 0.1088850274682045, "median": 0.1090676486492157, "p05": 0.08014046400785446, "p95": 0.13653376698493958 }, { "label": "completed_group_2", "mean": 0.05779466778039932, "median": 0.05755443871021271, "p05": 0.04364454001188278, "p95": 0.07319164276123047 }, { "label": "completed_group_3", "mean": 0.06273135542869568, "median": 0.06075543537735939, "p05": 0.04851231724023819, "p95": 0.07996035367250443 }, { "label": "completed_group_4", "mean": 0.07066236436367035, "median": 0.06902407109737396, "p05": 0.052817195653915405, "p95": 0.09365298599004745 }, { "label": "completed_group_5", "mean": 0.08129343390464783, "median": 0.08064926415681839, "p05": 0.06491829454898834, "p95": 0.09953147917985916 }, { "label": "completed_group_6", "mean": 0.17753681540489197, "median": 0.17456364631652832, "p05": 0.11964619904756546, "p95": 0.24453631043434143 }, { "label": "current_group_partial", "mean": 0.32435470819473267, "median": 0.32046452164649963, "p05": 0.19449321925640106, "p95": 0.4612511992454529 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.5923268795013428, "group": 7, "latest_source_mass": 0.4404740333557129, "layer": 26, "max_source_mass": 0.4404740333557129, "mixer_index": 51, "normalized_entropy": 0.7657473641769285, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.08656083047389984, "median": 0.0795329213142395, "p05": 0.050312869250774384, "p95": 0.14976806938648224 }, { "label": "completed_group_1", "mean": 0.05770102143287659, "median": 0.056408509612083435, "p05": 0.04085284471511841, "p95": 0.07934759557247162 }, { "label": "completed_group_2", "mean": 0.05105586349964142, "median": 0.049396127462387085, "p05": 0.034691184759140015, "p95": 0.07344280183315277 }, { "label": "completed_group_3", "mean": 0.039375390857458115, "median": 0.036708764731884, "p05": 0.024915706366300583, "p95": 0.05899723991751671 }, { "label": "completed_group_4", "mean": 0.04410157352685928, "median": 0.04195208102464676, "p05": 0.027429571375250816, "p95": 0.06846393644809723 }, { "label": "completed_group_5", "mean": 0.04500962793827057, "median": 0.043660376220941544, "p05": 0.030392050743103027, "p95": 0.06387433409690857 }, { "label": "completed_group_6", "mean": 0.2357216626405716, "median": 0.2248685657978058, "p05": 0.11950507014989853, "p95": 0.3818710744380951 }, { "label": "current_group_partial", "mean": 0.4404740333557129, "median": 0.4484519362449646, "p05": 0.24098733067512512, "p95": 0.6138508915901184 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.8469831943511963, "group": 7, "latest_source_mass": 0.33607518672943115, "layer": 27, "max_source_mass": 0.33607518672943115, "mixer_index": 52, "normalized_entropy": 0.8882111650319092, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.10824646055698395, "median": 0.11040513217449188, "p05": 0.05085912346839905, "p95": 0.16382771730422974 }, { "label": "completed_group_1", "mean": 0.09116974472999573, "median": 0.09135521948337555, "p05": 0.056257300078868866, "p95": 0.1266864538192749 }, { "label": "completed_group_2", "mean": 0.0701109915971756, "median": 0.06960364431142807, "p05": 0.05100060999393463, "p95": 0.0908425822854042 }, { "label": "completed_group_3", "mean": 0.0629766583442688, "median": 0.06064727157354355, "p05": 0.04317091405391693, "p95": 0.08672259747982025 }, { "label": "completed_group_4", "mean": 0.07049155235290527, "median": 0.06874367594718933, "p05": 0.050685517489910126, "p95": 0.09697030484676361 }, { "label": "completed_group_5", "mean": 0.07297641038894653, "median": 0.07156282663345337, "p05": 0.051110126078128815, "p95": 0.09974705427885056 }, { "label": "completed_group_6", "mean": 0.18795298039913177, "median": 0.1818247139453888, "p05": 0.1166272684931755, "p95": 0.27661576867103577 }, { "label": "current_group_partial", "mean": 0.33607518672943115, "median": 0.3355332612991333, "p05": 0.20117856562137604, "p95": 0.4717060327529907 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.6590619087219238, "group": 7, "latest_source_mass": 0.4179539084434509, "layer": 27, "max_source_mass": 0.4179539084434509, "mixer_index": 53, "normalized_entropy": 0.7978401294136326, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.09444728493690491, "median": 0.09060665965080261, "p05": 0.047914937138557434, "p95": 0.15662360191345215 }, { "label": "completed_group_1", "mean": 0.0654030293226242, "median": 0.06396090984344482, "p05": 0.04467916116118431, "p95": 0.09097769111394882 }, { "label": "completed_group_2", "mean": 0.060137998312711716, "median": 0.05839194357395172, "p05": 0.04035535454750061, "p95": 0.08539871126413345 }, { "label": "completed_group_3", "mean": 0.0449567511677742, "median": 0.04218417406082153, "p05": 0.02788582444190979, "p95": 0.06733109056949615 }, { "label": "completed_group_4", "mean": 0.044405773282051086, "median": 0.04241851717233658, "p05": 0.02826966717839241, "p95": 0.06812064349651337 }, { "label": "completed_group_5", "mean": 0.05244143307209015, "median": 0.05142951011657715, "p05": 0.035461653023958206, "p95": 0.07271309196949005 }, { "label": "completed_group_6", "mean": 0.2202538251876831, "median": 0.21207934617996216, "p05": 0.11892802268266678, "p95": 0.34954455494880676 }, { "label": "current_group_partial", "mean": 0.4179539084434509, "median": 0.4249612092971802, "p05": 0.2265779674053192, "p95": 0.586824893951416 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.884438157081604, "group": 7, "latest_source_mass": 0.2829248011112213, "layer": 28, "max_source_mass": 0.2829248011112213, "mixer_index": 54, "normalized_entropy": 0.9062231946945226, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.11613816022872925, "median": 0.12147872895002365, "p05": 0.04899268597364426, "p95": 0.1770799458026886 }, { "label": "completed_group_1", "mean": 0.09775452315807343, "median": 0.09778773784637451, "p05": 0.05219751223921776, "p95": 0.14079101383686066 }, { "label": "completed_group_2", "mean": 0.08394932001829147, "median": 0.08419600129127502, "p05": 0.05889316648244858, "p95": 0.10900868475437164 }, { "label": "completed_group_3", "mean": 0.07057419419288635, "median": 0.06822949647903442, "p05": 0.05265102535486221, "p95": 0.0926743820309639 }, { "label": "completed_group_4", "mean": 0.06920696794986725, "median": 0.06799854338169098, "p05": 0.05105053633451462, "p95": 0.09136901795864105 }, { "label": "completed_group_5", "mean": 0.06340071558952332, "median": 0.0628691017627716, "p05": 0.0437660738825798, "p95": 0.0860220268368721 }, { "label": "completed_group_6", "mean": 0.21605131030082703, "median": 0.21297892928123474, "p05": 0.14316481351852417, "p95": 0.2999357581138611 }, { "label": "current_group_partial", "mean": 0.2829248011112213, "median": 0.2790382504463196, "p05": 0.14648588001728058, "p95": 0.4251174330711365 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.7424441576004028, "group": 7, "latest_source_mass": 0.278835266828537, "layer": 28, "max_source_mass": 0.3030073344707489, "mixer_index": 55, "normalized_entropy": 0.8379385150653496, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.08763164281845093, "median": 0.05824008211493492, "p05": 0.027796253561973572, "p95": 0.2880856990814209 }, { "label": "completed_group_1", "mean": 0.07771545648574829, "median": 0.07654327154159546, "p05": 0.046035416424274445, "p95": 0.11438068747520447 }, { "label": "completed_group_2", "mean": 0.07502467930316925, "median": 0.07301308214664459, "p05": 0.04458038881421089, "p95": 0.11157117038965225 }, { "label": "completed_group_3", "mean": 0.06006643548607826, "median": 0.05717916786670685, "p05": 0.036717794835567474, "p95": 0.08720126748085022 }, { "label": "completed_group_4", "mean": 0.06314835697412491, "median": 0.061049528419971466, "p05": 0.03865773230791092, "p95": 0.09540070593357086 }, { "label": "completed_group_5", "mean": 0.054570816457271576, "median": 0.05328378826379776, "p05": 0.03375566378235817, "p95": 0.07989827543497086 }, { "label": "completed_group_6", "mean": 0.3030073344707489, "median": 0.30074357986450195, "p05": 0.14776623249053955, "p95": 0.469026654958725 }, { "label": "current_group_partial", "mean": 0.278835266828537, "median": 0.26309943199157715, "p05": 0.10582277178764343, "p95": 0.5049530863761902 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.8953055143356323, "group": 8, "latest_source_mass": 0.18233531713485718, "layer": 29, "max_source_mass": 0.2779633104801178, "mixer_index": 56, "normalized_entropy": 0.911449288833841, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.1135263666510582, "median": 0.0953834131360054, "p05": 0.02703639306128025, "p95": 0.31321749091148376 }, { "label": "completed_group_1", "mean": 0.10274903476238251, "median": 0.10363410413265228, "p05": 0.06506763398647308, "p95": 0.13818706572055817 }, { "label": "completed_group_2", "mean": 0.0858360230922699, "median": 0.08501487225294113, "p05": 0.059830889105796814, "p95": 0.11487221717834473 }, { "label": "completed_group_3", "mean": 0.08475606143474579, "median": 0.08265408128499985, "p05": 0.05679357051849365, "p95": 0.11555421352386475 }, { "label": "completed_group_4", "mean": 0.08078759908676147, "median": 0.07859720289707184, "p05": 0.055059194564819336, "p95": 0.11340422928333282 }, { "label": "completed_group_5", "mean": 0.07204629480838776, "median": 0.07071015238761902, "p05": 0.0498509518802166, "p95": 0.09922727942466736 }, { "label": "completed_group_6", "mean": 0.2779633104801178, "median": 0.2749147415161133, "p05": 0.15705733001232147, "p95": 0.39374127984046936 }, { "label": "completed_group_7", "mean": 0.18233531713485718, "median": 0.16221198439598083, "p05": 0.07201667875051498, "p95": 0.35018864274024963 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.801485300064087, "group": 8, "latest_source_mass": 0.3215197026729584, "layer": 29, "max_source_mass": 0.3215197026729584, "mixer_index": 57, "normalized_entropy": 0.8198912931549752, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.07824120670557022, "median": 0.06902759522199631, "p05": 0.017845120280981064, "p95": 0.17217300832271576 }, { "label": "completed_group_1", "mean": 0.060231711715459824, "median": 0.05764210224151611, "p05": 0.03765189275145531, "p95": 0.0922410786151886 }, { "label": "completed_group_2", "mean": 0.054100219160318375, "median": 0.0524706169962883, "p05": 0.034336626529693604, "p95": 0.07949143648147583 }, { "label": "completed_group_3", "mean": 0.039871927350759506, "median": 0.037151023745536804, "p05": 0.0235070139169693, "p95": 0.060298819094896317 }, { "label": "completed_group_4", "mean": 0.04207881912589073, "median": 0.040273066610097885, "p05": 0.021681729704141617, "p95": 0.06873302906751633 }, { "label": "completed_group_5", "mean": 0.045095667243003845, "median": 0.044285234063863754, "p05": 0.025416042655706406, "p95": 0.06805071234703064 }, { "label": "completed_group_6", "mean": 0.2047339677810669, "median": 0.1937887817621231, "p05": 0.08607597649097443, "p95": 0.3561563789844513 }, { "label": "completed_group_7", "mean": 0.15412676334381104, "median": 0.13271009922027588, "p05": 0.05149175971746445, "p95": 0.32324373722076416 }, { "label": "current_group_partial", "mean": 0.3215197026729584, "median": 0.31218427419662476, "p05": 0.08214674144983292, "p95": 0.5716250538825989 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.008213520050049, "group": 8, "latest_source_mass": 0.1996053159236908, "layer": 30, "max_source_mass": 0.21496367454528809, "mixer_index": 58, "normalized_entropy": 0.9139773606959576, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.09438058733940125, "median": 0.09028880298137665, "p05": 0.02690170146524906, "p95": 0.18525993824005127 }, { "label": "completed_group_1", "mean": 0.07153058052062988, "median": 0.070882648229599, "p05": 0.041449472308158875, "p95": 0.10379751026630402 }, { "label": "completed_group_2", "mean": 0.07423336803913116, "median": 0.0730072408914566, "p05": 0.048166655004024506, "p95": 0.10590565949678421 }, { "label": "completed_group_3", "mean": 0.06403130292892456, "median": 0.06157837435603142, "p05": 0.04371092468500137, "p95": 0.08951176702976227 }, { "label": "completed_group_4", "mean": 0.0645662397146225, "median": 0.0623253732919693, "p05": 0.043447867035865784, "p95": 0.09214113652706146 }, { "label": "completed_group_5", "mean": 0.05577655881643295, "median": 0.055099695920944214, "p05": 0.03416105732321739, "p95": 0.079859159886837 }, { "label": "completed_group_6", "mean": 0.21496367454528809, "median": 0.2118406593799591, "p05": 0.13403329253196716, "p95": 0.31010040640830994 }, { "label": "completed_group_7", "mean": 0.1609123945236206, "median": 0.15419161319732666, "p05": 0.07973575592041016, "p95": 0.27241089940071106 }, { "label": "current_group_partial", "mean": 0.1996053159236908, "median": 0.19533705711364746, "p05": 0.0843900740146637, "p95": 0.33647382259368896 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 1.8966764211654663, "group": 8, "latest_source_mass": 0.21615305542945862, "layer": 30, "max_source_mass": 0.21615305542945862, "mixer_index": 59, "normalized_entropy": 0.8632146393815058, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.09610478579998016, "median": 0.0941460132598877, "p05": 0.030834276229143143, "p95": 0.17860566079616547 }, { "label": "completed_group_1", "mean": 0.07267358899116516, "median": 0.06876465678215027, "p05": 0.03731950744986534, "p95": 0.11969097703695297 }, { "label": "completed_group_2", "mean": 0.060228411108255386, "median": 0.05793646723031998, "p05": 0.0349978432059288, "p95": 0.09366682916879654 }, { "label": "completed_group_3", "mean": 0.046177469193935394, "median": 0.04286886006593704, "p05": 0.02664034441113472, "p95": 0.07394801080226898 }, { "label": "completed_group_4", "mean": 0.04206448048353195, "median": 0.03912021219730377, "p05": 0.022682487964630127, "p95": 0.07153408974409103 }, { "label": "completed_group_5", "mean": 0.04096413776278496, "median": 0.039336949586868286, "p05": 0.023794004693627357, "p95": 0.06405855715274811 }, { "label": "completed_group_6", "mean": 0.21562576293945312, "median": 0.20660808682441711, "p05": 0.10648336261510849, "p95": 0.3508927822113037 }, { "label": "completed_group_7", "mean": 0.21000832319259644, "median": 0.204543337225914, "p05": 0.08861437439918518, "p95": 0.3495482802391052 }, { "label": "current_group_partial", "mean": 0.21615305542945862, "median": 0.19288693368434906, "p05": 0.07421350479125977, "p95": 0.43364205956459045 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 1.981720209121704, "group": 8, "latest_source_mass": 0.17098912596702576, "layer": 31, "max_source_mass": 0.23084521293640137, "mixer_index": 60, "normalized_entropy": 0.9019197352708571, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.10501759499311447, "median": 0.10529834032058716, "p05": 0.012508269399404526, "p95": 0.22564347088336945 }, { "label": "completed_group_1", "mean": 0.06946458667516708, "median": 0.06936883926391602, "p05": 0.025395508855581284, "p95": 0.11417273432016373 }, { "label": "completed_group_2", "mean": 0.07816962152719498, "median": 0.07627303898334503, "p05": 0.042986176908016205, "p95": 0.11851722747087479 }, { "label": "completed_group_3", "mean": 0.07385894656181335, "median": 0.07061562687158585, "p05": 0.04739787429571152, "p95": 0.1065390408039093 }, { "label": "completed_group_4", "mean": 0.06364607810974121, "median": 0.06080539897084236, "p05": 0.0418400801718235, "p95": 0.09455618262290955 }, { "label": "completed_group_5", "mean": 0.05328303202986717, "median": 0.05173730105161667, "p05": 0.03466875106096268, "p95": 0.07786844670772552 }, { "label": "completed_group_6", "mean": 0.23084521293640137, "median": 0.2265024036169052, "p05": 0.1294252872467041, "p95": 0.3483029305934906 }, { "label": "completed_group_7", "mean": 0.1547257900238037, "median": 0.15095551311969757, "p05": 0.057022616267204285, "p95": 0.2705056667327881 }, { "label": "current_group_partial", "mean": 0.17098912596702576, "median": 0.14655888080596924, "p05": 0.04728491231799126, "p95": 0.3703039288520813 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.0028607845306396, "group": 8, "latest_source_mass": 0.12469469010829926, "layer": 31, "max_source_mass": 0.2620389461517334, "mixer_index": 61, "normalized_entropy": 0.911541225776195, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.0842779129743576, "median": 0.06333573907613754, "p05": 0.03144961595535278, "p95": 0.24860140681266785 }, { "label": "completed_group_1", "mean": 0.10298711061477661, "median": 0.10017631947994232, "p05": 0.06670551002025604, "p95": 0.14914412796497345 }, { "label": "completed_group_2", "mean": 0.09207960218191147, "median": 0.09137493371963501, "p05": 0.056820835918188095, "p95": 0.13043460249900818 }, { "label": "completed_group_3", "mean": 0.07771934568881989, "median": 0.0746275931596756, "p05": 0.04656440392136574, "p95": 0.11502407491207123 }, { "label": "completed_group_4", "mean": 0.06265503168106079, "median": 0.0610337033867836, "p05": 0.038839153945446014, "p95": 0.09315080940723419 }, { "label": "completed_group_5", "mean": 0.055270493030548096, "median": 0.0542943961918354, "p05": 0.035113874822854996, "p95": 0.07895103842020035 }, { "label": "completed_group_6", "mean": 0.2620389461517334, "median": 0.25993645191192627, "p05": 0.13434287905693054, "p95": 0.3996090292930603 }, { "label": "completed_group_7", "mean": 0.1382768601179123, "median": 0.12482025474309921, "p05": 0.058253247290849686, "p95": 0.2677440643310547 }, { "label": "current_group_partial", "mean": 0.12469469010829926, "median": 0.11286871135234833, "p05": 0.04615473002195358, "p95": 0.24487172067165375 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.0655057430267334, "group": 8, "latest_source_mass": 0.09924688190221786, "layer": 32, "max_source_mass": 0.21871674060821533, "mixer_index": 62, "normalized_entropy": 0.9400521750629723, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.10472239553928375, "median": 0.08473113924264908, "p05": 0.030345402657985687, "p95": 0.2588862180709839 }, { "label": "completed_group_1", "mean": 0.11792458593845367, "median": 0.11655837297439575, "p05": 0.07099610567092896, "p95": 0.17131033539772034 }, { "label": "completed_group_2", "mean": 0.0928182601928711, "median": 0.09008213877677917, "p05": 0.06447342038154602, "p95": 0.12974750995635986 }, { "label": "completed_group_3", "mean": 0.10183011740446091, "median": 0.10004840791225433, "p05": 0.0707390084862709, "p95": 0.1374078094959259 }, { "label": "completed_group_4", "mean": 0.08287280797958374, "median": 0.08090724050998688, "p05": 0.05679214745759964, "p95": 0.11534547060728073 }, { "label": "completed_group_5", "mean": 0.06570011377334595, "median": 0.06424097716808319, "p05": 0.04414675012230873, "p95": 0.09294299781322479 }, { "label": "completed_group_6", "mean": 0.21871674060821533, "median": 0.21457019448280334, "p05": 0.13963721692562103, "p95": 0.3112483620643616 }, { "label": "completed_group_7", "mean": 0.11616809666156769, "median": 0.10701116174459457, "p05": 0.05174187570810318, "p95": 0.21264401078224182 }, { "label": "current_group_partial", "mean": 0.09924688190221786, "median": 0.08725139498710632, "p05": 0.032173603773117065, "p95": 0.20341788232326508 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 2.0258567333221436, "group": 8, "latest_source_mass": 0.12343063950538635, "layer": 32, "max_source_mass": 0.22868779301643372, "mixer_index": 63, "normalized_entropy": 0.9220071330979595, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.09674200415611267, "median": 0.07686861604452133, "p05": 0.04638649895787239, "p95": 0.2122267782688141 }, { "label": "completed_group_1", "mean": 0.09482249617576599, "median": 0.09244489669799805, "p05": 0.06637479364871979, "p95": 0.13019198179244995 }, { "label": "completed_group_2", "mean": 0.08996410667896271, "median": 0.08937586098909378, "p05": 0.06193297728896141, "p95": 0.12055040895938873 }, { "label": "completed_group_3", "mean": 0.06836152076721191, "median": 0.06606560945510864, "p05": 0.04674140736460686, "p95": 0.09358662366867065 }, { "label": "completed_group_4", "mean": 0.06293784081935883, "median": 0.061281464993953705, "p05": 0.04231695458292961, "p95": 0.08961650729179382 }, { "label": "completed_group_5", "mean": 0.05016915872693062, "median": 0.049648597836494446, "p05": 0.03570414334535599, "p95": 0.06624776124954224 }, { "label": "completed_group_6", "mean": 0.22868779301643372, "median": 0.2242993861436844, "p05": 0.1197199895977974, "p95": 0.3530009686946869 }, { "label": "completed_group_7", "mean": 0.1848844289779663, "median": 0.17580115795135498, "p05": 0.10029035806655884, "p95": 0.3003215491771698 }, { "label": "current_group_partial", "mean": 0.12343063950538635, "median": 0.10480014234781265, "p05": 0.05131315439939499, "p95": 0.2450288087129593 } ], "sources": 9 }, { "branch": "output", "entropy_mean": 1.7394249439239502, "group": null, "latest_source_mass": 0.2556654214859009, "layer": null, "max_source_mass": 0.3314756751060486, "mixer_index": null, "normalized_entropy": 0.7916464078663832, "offset": null, "source_summaries": [ { "label": "embedding", "mean": 0.10602210462093353, "median": 0.10817556083202362, "p05": 0.03750653192400932, "p95": 0.16905617713928223 }, { "label": "completed_group_1", "mean": 0.05902107432484627, "median": 0.05735805630683899, "p05": 0.039041999727487564, "p95": 0.08384283632040024 }, { "label": "completed_group_2", "mean": 0.049884356558322906, "median": 0.04808305203914642, "p05": 0.03180687129497528, "p95": 0.07381068915128708 }, { "label": "completed_group_3", "mean": 0.034865088760852814, "median": 0.03287554159760475, "p05": 0.02153801918029785, "p95": 0.05384448915719986 }, { "label": "completed_group_4", "mean": 0.03282830864191055, "median": 0.0315137580037117, "p05": 0.020427687093615532, "p95": 0.050021201372146606 }, { "label": "completed_group_5", "mean": 0.043302543461322784, "median": 0.041625380516052246, "p05": 0.027595922350883484, "p95": 0.06415434926748276 }, { "label": "completed_group_6", "mean": 0.08693542331457138, "median": 0.07748770713806152, "p05": 0.03693632781505585, "p95": 0.17101936042308807 }, { "label": "completed_group_7", "mean": 0.3314756751060486, "median": 0.32931262254714966, "p05": 0.0907490998506546, "p95": 0.5930418968200684 }, { "label": "completed_group_8", "mean": 0.2556654214859009, "median": 0.23854607343673706, "p05": 0.04562881588935852, "p95": 0.5189820528030396 } ], "sources": 9 } ], "mode": "learned", "optimizer_state_after": "a2cd8554c7844d96c2a16d3ea28050febab1d8839eabf8d6f91738dec12987fa", "optimizer_state_before": "a2cd8554c7844d96c2a16d3ea28050febab1d8839eabf8d6f91738dec12987fa", "optimizer_state_unchanged": true, "positions": { "attention_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "72af7516114c44d094899f6454538e045d6b49b293ae6fbf480fec93eec73e51", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.08528144657611847, 0.2063225358724594, 0.07479097694158554, 0.053203217685222626, 0.1070452407002449, 0.03943018242716789, 0.041509829461574554, 0.06257687509059906, 0.061002109199762344, 0.04113767296075821, 0.040399398654699326, 0.03907898813486099, 0.06714949756860733, 0.04437125846743584, 0.0412614606320858, 0.046277787536382675, 0.10073095560073853, 0.05776162073016167, 0.05185067281126976, 0.07827401161193848, 0.04170985147356987, 0.041845910251140594, 0.038240060210227966, 0.046402644366025925, 0.030182205140590668, 0.025695007294416428, 0.024306239560246468, 0.017157377675175667, 0.030125634744763374, 0.01829664409160614, 0.024464696645736694, 0.032671861350536346 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.1464898420854297e-05, "first_to_last_ratio": 0.8455323660117368, "imbalance_abs_log_ratio": 0.16778883105478307, "last_quartile_mean": 1.355938445612992e-05, "mean": 1.3058880313110421e-05, "normalized": [ 0.8914129879074366, 1.077571175585955, 0.9537440677326601, 0.8574453930361242, 0.9048950001587052, 0.8425968627438281, 0.7839582579962869, 0.7118870894358409, 0.5788084225246481, 0.530693141917732, 0.5023627734883869, 0.4844869540829288, 0.6659476097626051, 0.6212422516778212, 0.5836177358524666, 0.5530522631862488, 0.6705030645582619, 0.6390296210597277, 0.6539820627182751, 0.682372291616315, 3.0705747439001825, 2.5957154007666214, 2.1136981524200404, 1.7237888584415353, 1.9773452676094638, 1.7509357725203292, 1.3561771885891878, 0.9354730786944212, 0.8232148160475435, 0.5856284416831322, 0.45203530207589593, 0.4258039502093924 ], "peak_layer": 21, "peak_normalized": 3.0705747439001825, "population_cv": 0.6442856671110848, "rest_mean": 9.924206164281781e-06, "spike_contrast": 3.0215132797934157, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.998612071678508e-05, "top_five_layers": [ 21, 22, 23, 25, 26 ], "top_five_spike_overlap": 4 }, "values": [ 1.1640855518635362e-05, 1.4071873010834679e-05, 1.2454829629859887e-05, 1.119727676268667e-05, 1.1816915503004566e-05, 1.100337158277398e-05, 1.0237617061648052e-05, 9.29644829739118e-06, 7.558589913969627e-06, 6.930258223292185e-06, 6.560295332747046e-06, 6.326857146632392e-06, 8.696530130691826e-06, 8.112728210107889e-06, 7.6213941611058544e-06, 7.222243311844068e-06, 8.756019269640092e-06, 8.345011337951291e-06, 8.540273483959027e-06, 8.91101808520034e-06, 4.009826807305217e-05, 3.389713674550876e-05, 2.7602531190495938e-05, 2.2510752387461253e-05, 2.5821915187407285e-05, 2.2865260689286515e-05, 1.7710155589156784e-05, 1.2216230970807374e-05, 1.0750263754744083e-05, 7.64765172789339e-06, 5.90307490710984e-06, 5.560522822634084e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 4.568297117657494e-05, "first_to_last_ratio": 0.8467277881503474, "imbalance_abs_log_ratio": 0.16637601951453293, "last_quartile_mean": 5.3952370308252284e-05, "mean": 5.2044935273443116e-05, "normalized": [ 0.8954256553649687, 1.0662758237422765, 0.9521993752570108, 0.8636540973713456, 0.9057610370429903, 0.8375327549345261, 0.7860647798171227, 0.7151678277570597, 0.5850341337918961, 0.5346127123576437, 0.503507273341017, 0.4847402912324663, 0.6665718638058081, 0.6225508325294531, 0.5847778078263528, 0.5518762696320233, 0.6703631394314256, 0.6385628522390465, 0.6481962908175356, 0.6867108224315941, 3.0618207183884687, 2.6065455407648237, 2.120445260989786, 1.7184046595544948, 1.970749079693583, 1.7371379145623163, 1.3426067482179225, 0.9360274925849964, 0.8279776750708923, 0.5898304420688857, 0.45715187233946364, 0.4317169550408049 ], "peak_layer": 21, "peak_normalized": 3.0618207183884687, "population_cv": 0.6425897668413451, "rest_mean": 3.955807295386438e-05, "spike_contrast": 3.0202176920627024, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00011947399179916829, "top_five_layers": [ 21, 22, 23, 25, 26 ], "top_five_spike_overlap": 4 }, "values": [ 4.6602370275650173e-05, 5.549425623030402e-05, 4.95571548526641e-05, 4.494882159633562e-05, 4.714027454610914e-05, 4.35893380199559e-05, 4.091069058631547e-05, 3.7220863305265084e-05, 3.044806362595409e-05, 2.7823884011013433e-05, 2.620500345074106e-05, 2.5228277081623673e-05, 3.4691689506871626e-05, 3.240061778342351e-05, 3.0434723157668486e-05, 2.872236473194789e-05, 3.4889006201410666e-05, 3.32339623128064e-05, 3.373533400008455e-05, 3.5739820305025205e-05, 0.00015935226110741496, 0.00013565749395638704, 0.00011035843635909259, 8.943425928009674e-05, 0.00010256750829285011, 9.04092303244397e-05, 6.987588130868971e-05, 4.8715490265749395e-05, 4.30920445069205e-05, 3.0697687179781497e-05, 2.3792439606040716e-05, 2.246868098154664e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.040512115228921175, "first_to_last_ratio": 0.8467277882597376, "imbalance_abs_log_ratio": 0.16637601938534127, "last_quartile_mean": 0.047845500987023115, "mean": 0.046153969364240766, "normalized": [ 0.8954256214236297, 1.0662758259809275, 0.9521993286286553, 0.8636540898502394, 0.9057610206102777, 0.8375327282419698, 0.7860647826245731, 0.7151678297957529, 0.5850341696121162, 0.5346127266472548, 0.5035072907230911, 0.484740303083307, 0.666571864231768, 0.6225508316512409, 0.5847778206533832, 0.5518762470155281, 0.6703631006740365, 0.6385628347715187, 0.6481963011119384, 0.6867108353729822, 3.0618206630891383, 2.6065457195235173, 2.120445297956706, 1.7184047348201537, 1.9707489675025585, 1.7371379031231275, 1.3426067663321783, 0.9360274706240039, 0.8279776759485932, 0.5898304215013819, 0.45715187640138183, 0.4317169504730694 ], "peak_layer": 21, "peak_normalized": 3.0618206630891383, "population_cv": 0.6425897745941355, "rest_mean": 0.03508049470406992, "spike_contrast": 3.0202177427352903, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.10595073252916336, "top_five_layers": [ 21, 22, 23, 25, 26 ], "top_five_spike_overlap": 4 }, "values": [ 0.041327446699142456, 0.04921286180615425, 0.04394777864217758, 0.03986106440424919, 0.041804466396570206, 0.03865545988082886, 0.036280009895563126, 0.03300783410668373, 0.027001649141311646, 0.02467449940741062, 0.023238860070705414, 0.022372689098119736, 0.03076493740081787, 0.02873319201171398, 0.02698981761932373, 0.02547127939760685, 0.030939918011426926, 0.02947220951318741, 0.029916832223534584, 0.031694430857896805, 0.14131517708301544, 0.12030243128538132, 0.0978669673204422, 0.07931119948625565, 0.0909578874707222, 0.08017580956220627, 0.061966631561517715, 0.04320138320326805, 0.03821445629000664, 0.027223015204072, 0.021099373698234558, 0.019925450906157494 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 1.8178302667593016e-06, "first_to_last_ratio": 0.5015852279301855, "imbalance_abs_log_ratio": 0.6899817399970322, "last_quartile_mean": 3.6241702616734983e-06, "mean": 2.789410469716813e-06, "normalized": [ 0.8037687132877044, 0.7183737285046393, 0.691319672118338, 0.6807254875963729, 0.6052756387601809, 0.5818941580160492, 0.565889251881305, 0.5662715078678539, 0.4369805018739323, 0.41687613565271625, 0.40783237228271835, 0.41545507589336067, 0.5056083509177715, 0.5034210266522742, 0.5059866535158252, 0.503281516870639, 0.5409761795262856, 0.5584199152903219, 0.570327765976957, 0.6037643031707576, 2.871505340690978, 2.8594035701851728, 2.496246607238993, 2.1963141894218743, 2.393616152828075, 2.14842944375899, 1.7201180299928378, 1.2992827591669334, 1.010592310992817, 0.7253855724579202, 0.5639386418465228, 0.532719425762883 ], "peak_layer": 21, "peak_normalized": 2.871505340690978, "population_cv": 0.7722487160005334, "rest_mean": 1.9818156126213445e-06, "spike_contrast": 3.6080161305089784, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 7.150422698032344e-06, "top_five_layers": [ 21, 22, 23, 25, 24 ], "top_five_spike_overlap": 5 }, "values": [ 2.242040864075534e-06, 2.0038391994603444e-06, 1.9283743313280866e-06, 1.8988228021044051e-06, 1.6883622038221802e-06, 1.6231416566370171e-06, 1.5784974038979271e-06, 1.5795636727489182e-06, 1.2189179869892541e-06, 1.162838657364773e-06, 1.1376118891348597e-06, 1.1588747383939335e-06, 1.4103492276262841e-06, 1.4042478824194404e-06, 1.4114044688540162e-06, 1.4038587323739193e-06, 1.5090046190380235e-06, 1.5576623582091997e-06, 1.5908782415863243e-06, 1.6841464685057872e-06, 8.009807061171159e-06, 7.976050255820155e-06, 6.96305642122752e-06, 6.126421794760972e-06, 6.676777957181912e-06, 5.9928515838691965e-06, 4.798115242010681e-06, 3.6242329315427924e-06, 2.818956772898673e-06, 2.0233981103956467e-06, 1.5730563518445706e-06, 1.4859731436445145e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 3.489204118523048e-05, "first_to_last_ratio": 1.0102007473277157, "imbalance_abs_log_ratio": 0.01014907083348142, "last_quartile_mean": 3.45397103274081e-05, "mean": 3.719139070312849e-05, "normalized": [ 0.978746778311648, 1.126838378619792, 1.0180419362177582, 0.9373857091569082, 0.9573174487244173, 0.8927610480088312, 0.8376961987226147, 0.7566142687378048, 0.6421710022561828, 0.5898929811647327, 0.5545314669590201, 0.532480114188549, 0.7191190555706168, 0.6685869838104541, 0.6276517153974418, 0.5907065315366481, 0.7404063048506205, 0.7009976783342637, 0.7076013069062036, 0.7352296418703005, 3.079471174742761, 2.569668642671924, 2.031249327096867, 1.5752201563285502, 1.8426406293315767, 1.5558607485184148, 1.2001391008391664, 0.8699295985429139, 0.6815919538112388, 0.4881760018644189, 0.4044368033178356, 0.3868393135895246 ], "peak_layer": 21, "peak_normalized": 3.079471174742761, "population_cv": 0.6121845820849332, "rest_mean": 2.8791301971338084e-05, "spike_contrast": 2.867250322231956, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 8.255186985479668e-05, "top_five_layers": [ 21, 22, 23, 25, 24 ], "top_five_spike_overlap": 5 }, "values": [ 3.640095383161679e-05, 4.1908686398528516e-05, 3.786239540204406e-05, 3.486267814878374e-05, 3.560396726243198e-05, 3.320302494103089e-05, 3.115508661721833e-05, 2.8139536880189553e-05, 2.3883232643129304e-05, 2.1938940335530788e-05, 2.0623796444851905e-05, 1.98036759684328e-05, 2.674503775779158e-05, 2.4865679733920842e-05, 2.3343240172835067e-05, 2.196919740526937e-05, 2.7536740162759088e-05, 2.6071078536915593e-05, 2.631667666719295e-05, 2.7344212867319584e-05, 0.00011452981561888009, 9.55695504671894e-05, 7.554498733952641e-05, 5.858462827745825e-05, 6.853036757092923e-05, 5.786462497781031e-05, 4.463484219741076e-05, 3.235389158362523e-05, 2.5349352654302493e-05, 1.8155944417230785e-05, 1.5041567166917957e-05, 1.4387092051038053e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 2.6185193746641744e-05, "first_to_last_ratio": 1.1813045118653667, "imbalance_abs_log_ratio": 0.16661934637193965, "last_quartile_mean": 2.2166336862028402e-05, "mean": 2.6416844121968097e-05, "normalized": [ 1.0582483794330777, 1.1839113566742743, 1.0890809034246702, 1.013584100454415, 1.0023428903115454, 0.935888205711016, 0.8664359614724408, 0.7803558787188484, 0.6948576391403528, 0.6454578452882075, 0.6088646607200737, 0.5814486923097771, 0.764934624577467, 0.7118430529936276, 0.6645388585723574, 0.629085742341511, 0.8117172258565646, 0.7682825984070001, 0.7655811246450389, 0.7857710139637165, 3.1099202343035985, 2.532060860240954, 1.9118190653037337, 1.3711803024378004, 1.6503311681253898, 1.3613514907810353, 1.0865408308937656, 0.8606461743085547, 0.5744935662233659, 0.4301536296331042, 0.37929000098103194, 0.3699819217516835 ], "peak_layer": 21, "peak_normalized": 3.1099202343035985, "population_cv": 0.583027433434022, "rest_mean": 2.096195011189492e-05, "spike_contrast": 2.6654615376008266, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.5873271776363254e-05, "top_five_layers": [ 21, 22, 23, 25, 24 ], "top_five_spike_overlap": 5 }, "values": [ 2.795558248180896e-05, 3.127520176349208e-05, 2.8770080461981706e-05, 2.6775693186209537e-05, 2.6478735890123062e-05, 2.4723212845856324e-05, 2.2888503735885024e-05, 2.061453960777726e-05, 1.835594594012946e-05, 1.7050959286279976e-05, 1.608428283361718e-05, 1.5360039469669573e-05, 2.020715874095913e-05, 1.8804646970238537e-05, 1.755501943989657e-05, 1.661845999478828e-05, 2.1443007426569238e-05, 2.0295601643738337e-05, 2.0224237232469022e-05, 2.0757590391440317e-05, 8.215427806135267e-05, 6.688905705232173e-05, 5.050422623753548e-05, 3.6222256312612444e-05, 4.3596541217993945e-05, 3.59626101271715e-05, 2.8702979761874303e-05, 2.2735555830877274e-05, 1.5176306987996213e-05, 1.136330138251651e-05, 1.0019644832937047e-05, 9.77375475486042e-06 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 9.068558847502572e-05, "first_to_last_ratio": 0.7914793598068353, "imbalance_abs_log_ratio": 0.2338514773139948, "last_quartile_mean": 0.00011457732580311131, "mean": 0.00010754580421235005, "normalized": [ 0.8538738720411836, 1.0095472825344882, 0.8946236658075398, 0.8160443171140711, 0.8628188117624934, 0.8243645351821149, 0.7801447421092808, 0.7044034320375209, 0.5665887799168225, 0.525608047514618, 0.4958157795364288, 0.4792984745903937, 0.6722545548598279, 0.6294814179611099, 0.5936561952518512, 0.5594680423070397, 0.6646720414600984, 0.6282112723775689, 0.6412680496471301, 0.6738175767739937, 3.007412745002923, 2.616093975094252, 2.1689004360837822, 1.8085787931443134, 2.061415837562002, 1.8302883103495597, 1.4233838379602504, 0.9902275928882596, 0.8088632655175125, 0.5559016015107706, 0.43871981288186035, 0.4142529012189394 ], "peak_layer": 21, "peak_normalized": 3.007412745002923, "population_cv": 0.6573728454605334, "rest_mean": 8.100827242877265e-05, "spike_contrast": 3.096578513808316, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.000250848475843668, "top_five_layers": [ 21, 22, 23, 25, 26 ], "top_five_spike_overlap": 4 }, "values": [ 9.183055226458237e-05, 0.00010857257439056411, 9.621302160667256e-05, 8.776214235695079e-05, 9.279254300054163e-05, 8.865694690030068e-05, 8.390129369217902e-05, 7.575563358841464e-05, 6.093424599384889e-05, 5.6526940170442685e-05, 5.3322906751418486e-05, 5.1546539907576516e-05, 7.229815673781559e-05, 6.769808533135802e-05, 6.384523294400424e-05, 6.016844054101966e-05, 7.148268923629075e-05, 6.756148650310934e-05, 6.896568811498582e-05, 7.246625318657607e-05, 0.00032343462225981057, 0.00028134993044659495, 0.00023325614165514708, 0.00019450506079010665, 0.00022169662406668067, 0.00019683982827700675, 0.00015307895955629647, 0.00010649482283042744, 8.698985038790852e-05, 5.978488479740918e-05, 4.7182475100271404e-05, 4.455116140889004e-05 ] } } }, "mlp_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "16b9daa70ef1c39a83a37551a158c8705d864e7314a8382a1f918981811300d1", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.13266965746879578, 0.045500293374061584, 0.018729008734226227, 0.03815881907939911, 0.02209169790148735, 0.01640710048377514, 0.02340921014547348, 0.05481670796871185, 0.037687551230192184, 0.04149388521909714, 0.048693906515836716, 0.06190986558794975, 0.04065176844596863, 0.049332331866025925, 0.0589165985584259, 0.08173932880163193, 0.043543290346860886, 0.05228494107723236, 0.04403071105480194, 0.0687810555100441, 0.030229199677705765, 0.04699210822582245, 0.06082639843225479, 0.07667754590511322, 0.050203632563352585, 0.05539015680551529, 0.06946423649787903, 0.07630576938390732, 0.07239200174808502, 0.09628339111804962, 0.12468792498111725, 0.15056027472019196 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.1373777851986233e-05, "first_to_last_ratio": 1.0318498984683846, "imbalance_abs_log_ratio": 0.03135320926034069, "last_quartile_mean": 1.1022705791674525e-05, "mean": 1.1832125267119409e-05, "normalized": [ 0.9800012201604696, 1.1859002007221142, 1.0527348906308314, 0.9435939146843411, 0.9942658078795703, 0.9213667974504651, 0.8458121244297144, 0.7664247312681599, 0.6009152433386855, 0.5652338271700392, 0.5379286363125223, 0.5220099956402732, 0.7166619388130793, 0.6623381156132455, 0.6162294438595727, 0.5904010132472348, 0.7324356475968063, 0.6882143187892091, 0.7092129547489621, 0.8089361635707597, 3.115397457088153, 2.4321274256844663, 1.9642577727784198, 1.5948693957730091, 1.9522870367789025, 1.5037975945274742, 1.0368577963312928, 0.8807302172788337, 0.6474760441659657, 0.5016437023911682, 0.4711656138967313, 0.45877295737952767 ], "peak_layer": 21, "peak_normalized": 3.115397457088153, "population_cv": 0.603271891680543, "rest_mean": 9.176935405034951e-06, "spike_contrast": 2.851730928390011, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.617015052237548e-05, "top_five_layers": [ 21, 22, 23, 25, 24 ], "top_five_spike_overlap": 5 }, "values": [ 1.1595497198868543e-05, 1.4031719729246106e-05, 1.2456091099011246e-05, 1.1164721399836708e-05, 1.1764277587644756e-05, 1.0901727364398539e-05, 1.000775500870077e-05, 9.068433428183198e-06, 7.110104434104869e-06, 6.687917448289227e-06, 6.364839009620482e-06, 6.176487659104168e-06, 8.479633834213018e-06, 7.836867553123739e-06, 7.291303973033791e-06, 6.985698746575508e-06, 8.66627033246914e-06, 8.143038030539174e-06, 8.391496521653607e-06, 9.571434020472225e-06, 3.686177296913229e-05, 2.8777236366295256e-05, 2.3241344024427235e-05, 1.8870694475481287e-05, 2.309970477654133e-05, 1.7793121514841914e-05, 1.226823133038124e-05, 1.0420910257380456e-05, 7.661017662030645e-06, 5.9355111261538696e-06, 5.574890565185342e-06, 5.428259100881405e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 4.52927956757776e-05, "first_to_last_ratio": 1.0314200550157464, "imbalance_abs_log_ratio": 0.03093654690539193, "last_quartile_mean": 4.3913045374210924e-05, "mean": 4.717359485084671e-05, "normalized": [ 0.9832135136979314, 1.173164387917132, 1.0505394497504255, 0.9501840008496486, 0.9944561477628384, 0.9147691984159712, 0.8464742625690872, 0.7682410828301822, 0.6067449473404415, 0.5681699639617173, 0.5386389915171271, 0.5211401199814687, 0.7167243803511351, 0.6632392162921593, 0.6172595012574109, 0.588538933455825, 0.7318717018485573, 0.6893632656633977, 0.7026754642712614, 0.8079854981931587, 3.1150142181403937, 2.4538347722826446, 1.966137677039432, 1.5845641436792013, 1.9375330190421414, 1.4892879811658426, 1.035008152926599, 0.8840423036226741, 0.6517018335432763, 0.5065002027138268, 0.477654085237329, 0.4653275826797635 ], "peak_layer": 21, "peak_normalized": 3.1150142181403937, "population_cv": 0.6027267551758898, "rest_mean": 3.659083860704278e-05, "spike_contrast": 2.8509999371074524, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00010432047856738791, "top_five_layers": [ 21, 22, 23, 25, 24 ], "top_five_spike_overlap": 5 }, "values": [ 4.638171594706364e-05, 5.534238152904436e-05, 4.9557722377358004e-05, 4.482359508983791e-05, 4.691207141149789e-05, 4.315295154810883e-05, 3.993123391410336e-05, 3.6240693589206785e-05, 2.862234032363631e-05, 2.6802619686350226e-05, 2.5409537556697614e-05, 2.458405288052745e-05, 3.381046553840861e-05, 3.1287378078559414e-05, 2.9118349630152807e-05, 2.7763497200794518e-05, 3.4525019145803526e-05, 3.251974339946173e-05, 3.3147727663163096e-05, 3.8115580537123606e-05, 0.00014694641868118197, 0.00011575620737858117, 9.274978219764307e-05, 7.47495869291015e-05, 9.140039765043184e-05, 7.025506783975288e-05, 4.8825055273482576e-05, 4.1703453462105244e-05, 3.074311825912446e-05, 2.3893435354693793e-05, 2.253266029583756e-05, 2.1951174858259037e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.0401661042124033, "first_to_last_ratio": 1.0314200458773342, "imbalance_abs_log_ratio": 0.030936538045362262, "last_quartile_mean": 0.03894252819009125, "mean": 0.041834015981294215, "normalized": [ 0.9832134785727225, 1.1731643857154097, 1.0505394772834944, 0.9501839314909589, 0.9944561321819699, 0.9147691986147233, 0.8464742543512297, 0.7682411398349651, 0.6067449159473433, 0.5681700011232179, 0.5386389788463947, 0.5211401213699254, 0.7167244214988735, 0.663239179094363, 0.6172595023900571, 0.5885389178924195, 0.7318717092957638, 0.6893632192579286, 0.7026754233788964, 0.8079854502108175, 3.1150143563768617, 2.4538347987960454, 1.9661377249833973, 1.5845640989337733, 1.937532949114683, 1.4892880148885772, 1.0350081198083678, 0.8840423776654689, 0.6517018613150984, 0.5065001920223166, 0.4776540925208594, 0.46532757522307666 ], "peak_layer": 21, "peak_normalized": 3.1150143563768617, "population_cv": 0.6027267719407451, "rest_mean": 0.03244912169045872, "spike_contrast": 2.850999975725323, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.09251244515180587, "top_five_layers": [ 21, 22, 23, 25, 24 ], "top_five_spike_overlap": 5 }, "values": [ 0.04113176837563515, 0.04907817766070366, 0.04394828528165817, 0.03975000977516174, 0.04160209372639656, 0.03826846927404404, 0.03541141748428345, 0.03213861212134361, 0.025382576510310173, 0.02376883290708065, 0.02253343164920807, 0.021801384165883064, 0.029983460903167725, 0.027745958417654037, 0.025822443887591362, 0.024620946496725082, 0.030617132782936096, 0.028838831931352615, 0.029395734891295433, 0.03380127623677254, 0.13031356036663055, 0.10265376418828964, 0.08225143700838089, 0.06628867983818054, 0.08105478435754776, 0.06230289861559868, 0.043298546224832535, 0.03698304295539856, 0.027263306081295013, 0.02118893712759018, 0.019982188940048218, 0.019466521218419075 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 1.809132328389751e-06, "first_to_last_ratio": 0.5997744339081645, "imbalance_abs_log_ratio": 0.5112016379368517, "last_quartile_mean": 3.016354526153009e-06, "mean": 2.487152045205221e-06, "normalized": [ 0.890408531440174, 0.809673961899388, 0.7756270411999202, 0.7664197016433177, 0.6818557608603003, 0.6497146124011205, 0.6267527377763467, 0.6186766662305586, 0.4670749596929491, 0.4522141597899267, 0.449235216497964, 0.45731293359098185, 0.5550856819872789, 0.5467983404610478, 0.5410652082029204, 0.536255547908205, 0.6011915384635866, 0.6092812314836655, 0.6241321581033094, 0.6880744209704973, 2.915841858020288, 2.6264204515100484, 2.3560883070579246, 2.052603139729255, 2.421562802944245, 1.9231629319277221, 1.4565088666484294, 1.2845058483948424, 0.8068415180205192, 0.6295503966151642, 0.5969673730151487, 0.5830960955129549 ], "peak_layer": 21, "peak_normalized": 2.915841858020288, "population_cv": 0.7106082711854734, "rest_mean": 1.8080198363653228e-06, "spike_contrast": 3.4039814437617255, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.154465972940671e-06, "top_five_layers": [ 21, 22, 25, 23, 24 ], "top_five_spike_overlap": 5 }, "values": [ 2.214581400039606e-06, 2.013782250287477e-06, 1.9291023818368558e-06, 1.906202328427753e-06, 1.695878950158658e-06, 1.6159390270331642e-06, 1.5588293535984121e-06, 1.5387429357360816e-06, 1.1616864412644645e-06, 1.1247253723922768e-06, 1.1173162874911213e-06, 1.1374067980796099e-06, 1.3805824892187957e-06, 1.3599706107925158e-06, 1.3457114391712821e-06, 1.3337490827325382e-06, 1.4952547644497827e-06, 1.5153750609897543e-06, 1.5523115735049942e-06, 1.7113457033701707e-06, 7.252142040670151e-06, 6.532306997542037e-06, 5.859949851583224e-06, 5.105136096972274e-06, 6.022794877935667e-06, 4.783198619406903e-06, 3.6225590065441793e-06, 3.1947613479132997e-06, 2.0067375317012193e-06, 1.5657875565011636e-06, 1.484748622715415e-06, 1.4502486465062248e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 3.464051883383945e-05, "first_to_last_ratio": 1.2085017018903634, "imbalance_abs_log_ratio": 0.18938132942715577, "last_quartile_mean": 2.866402155632386e-05, "mean": 3.421727353725146e-05, "normalized": [ 1.059950361009556, 1.2224231923999602, 1.1066505484293667, 1.0166204096806029, 1.0363170309532028, 0.9614969179125695, 0.8913049122896795, 0.8041913955656627, 0.6595104635459774, 0.6198402043713972, 0.5858567866753434, 0.5655998592669403, 0.7633893605247403, 0.7040501232645258, 0.6548491311439005, 0.6226536428847976, 0.7974262571555777, 0.7481683838617732, 0.7573132817097481, 0.8440618552058439, 3.1217920974447857, 2.4268766121530727, 1.8727793487823996, 1.45522837174708, 1.7605498171411489, 1.332061600542984, 0.9562588529424351, 0.830045802534728, 0.533682078716892, 0.4444300159412478, 0.423319578555294, 0.4213017056467665 ], "peak_layer": 21, "peak_normalized": 3.1217920974447857, "population_cv": 0.5796313993689626, "rest_mean": 2.7073180481912225e-05, "spike_contrast": 2.6888372455803045, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 7.279537603608333e-05, "top_five_layers": [ 21, 22, 23, 25, 24 ], "top_five_spike_overlap": 5 }, "values": [ 3.626861143857241e-05, 4.182798875262961e-05, 3.7866564525756985e-05, 3.478597864159383e-05, 3.5459943319438025e-05, 3.28998030454386e-05, 3.0498023988911882e-05, 2.751723695837427e-05, 2.2566649931832217e-05, 2.1209241822361946e-05, 2.0046421923325397e-05, 1.9353285097167827e-05, 2.6121102564502507e-05, 2.4090675651677884e-05, 2.2407151845982298e-05, 2.1305510017555207e-05, 2.7285752366879024e-05, 2.5600282242521644e-05, 2.5913195713656023e-05, 2.8881495381938294e-05, 0.00010681921412469819, 8.30411008791998e-05, 6.408140325220302e-05, 4.979394725523889e-05, 6.0241214669076726e-05, 4.557951615424827e-05, 3.2720570743549615e-05, 2.84019042737782e-05, 1.826114566938486e-05, 1.5207183423626702e-05, 1.4484841813100502e-05, 1.4415795703826007e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 2.6040052716780338e-05, "first_to_last_ratio": 1.3601380691640932, "imbalance_abs_log_ratio": 0.3075862160391937, "last_quartile_mean": 1.9145153942190518e-05, "mean": 2.476174321941471e-05, "normalized": [ 1.1242415044070515, 1.2621466755459967, 1.1626111778912407, 1.0807522798319469, 1.0668131611583063, 0.9898612219145286, 0.9055963728518246, 0.8209725998576762, 0.7059309419194623, 0.6698532100359812, 0.6332217391849956, 0.605864404334209, 0.7980357739095889, 0.7372559068278083, 0.6856394720285257, 0.6565553942525388, 0.8578727127097449, 0.8064680621295278, 0.8069038983319957, 0.8750777012450611, 3.1375211320361442, 2.3884634889966896, 1.7586693778594196, 1.27827401077098, 1.559779974845964, 1.1976668654320728, 0.9315461470291763, 0.8389979859505794, 0.4605019751312927, 0.40913221789275306, 0.3951692615572719, 0.3926033521296457 ], "peak_layer": 21, "peak_normalized": 3.1375211320361442, "population_cv": 0.5588199044745877, "rest_mean": 2.0063699526805e-05, "spike_contrast": 2.4986009729926506, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.0131179159507154e-05, "top_five_layers": [ 21, 22, 23, 25, 24 ], "top_five_spike_overlap": 5 }, "values": [ 2.78381794487359e-05, 3.1252951885107905e-05, 2.8788279450964183e-05, 2.67613104369957e-05, 2.6416153559694067e-05, 2.451068939990364e-05, 2.2424144844990224e-05, 2.0328712707851082e-05, 1.7480080714449286e-05, 1.6586733181611635e-05, 1.5679674106650054e-05, 1.5002258805907331e-05, 1.9760756913456134e-05, 1.8255741451866925e-05, 1.697762854746543e-05, 1.6257456081802957e-05, 2.124242382706143e-05, 1.9969555069110356e-05, 1.9980347133241594e-05, 2.1668449335265905e-05, 7.769049261696637e-05, 5.914251960348338e-05, 4.3547719542402774e-05, 3.165229281876236e-05, 3.8622871215920895e-05, 2.96563193842303e-05, 2.3066706489771605e-05, 2.0775052689714357e-05, 1.140283166023437e-05, 1.013082692224998e-05, 9.785079782886896e-06, 9.721543392515741e-06 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 9.001170565170469e-05, "first_to_last_ratio": 0.9626745767731016, "imbalance_abs_log_ratio": 0.038039850801722765, "last_quartile_mean": 9.350169602839742e-05, "mean": 9.76294164729552e-05, "normalized": [ 0.9359776868957951, 1.1142596124196091, 0.9861551159867753, 0.8961521822246734, 0.949684039356357, 0.8995149572091795, 0.846411402631371, 0.747630614092772, 0.5888085074137973, 0.5541267695979373, 0.5294131458395803, 0.5184084456985816, 0.7260183643779575, 0.6675529417749002, 0.6222711070136198, 0.5862023232639773, 0.7253041044725564, 0.6816202250286443, 0.6942528027900181, 0.79595263681847, 3.0863426882796725, 2.474889704855176, 2.017495070776267, 1.693791348848989, 2.023091547543398, 1.5725916380980733, 1.090726894893622, 0.9642815025941395, 0.6146236296489728, 0.48585096781736253, 0.45630461264981964, 0.45429340908793464 ], "peak_layer": 21, "peak_normalized": 3.0863426882796725, "population_cv": 0.6173495462344238, "rest_mean": 7.486509181304697e-05, "spike_contrast": 2.9460562232025813, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00022055676963645966, "top_five_layers": [ 21, 22, 25, 23, 24 ], "top_five_spike_overlap": 5 }, "values": [ 9.137895540334284e-05, 0.00010878451575990766, 9.627774852560833e-05, 8.749081462156028e-05, 9.271709859604016e-05, 8.781912038102746e-05, 8.263465133495629e-05, 7.299074059119448e-05, 5.7485030993120745e-05, 5.409907316789031e-05, 5.168629650142975e-05, 5.06119140482042e-05, 7.088074926286936e-05, 6.517280417028815e-05, 6.075196506571956e-05, 5.723059075535275e-05, 7.081101648509502e-05, 6.654618482571095e-05, 6.777949602110311e-05, 7.770839147269726e-05, 0.0003013178356923163, 0.00024162203771993518, 0.0001969668664969504, 0.0001653638610150665, 0.00019751324725802988, 0.00015353120397776365, 0.00010648703027982265, 9.414224041393027e-05, 6.0005346313118935e-05, 4.743334648082964e-05, 4.4548753066919744e-05, 4.435240043676458e-05 ] } } }, "post_attention_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "90f1dd0e3ce75c2ca97d856d889fb391792298af07000622c435ff53f6d398a7", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.08528144657611847, 0.3133305609226227, 0.3510810136795044, 0.36267346143722534, 0.1070452407002449, 0.1260577291250229, 0.14502130448818207, 0.17329032719135284, 0.061002109199762344, 0.0987050011754036, 0.13416865468025208, 0.17569874227046967, 0.06714949756860733, 0.09735818952322006, 0.13312570750713348, 0.17941679060459137, 0.10073095560073853, 0.13780468702316284, 0.17965665459632874, 0.24453195929527283, 0.04170985147356987, 0.07352267950773239, 0.10462787002325058, 0.14256280660629272, 0.030182205140590668, 0.06915032118558884, 0.10093247890472412, 0.1392749696969986, 0.030125634744763374, 0.08267941325902939, 0.15948866307735443, 0.2670789062976837 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.1464836006780388e-05, "first_to_last_ratio": 0.8455234102400129, "imbalance_abs_log_ratio": 0.16779942298421527, "last_quartile_mean": 1.355945425984828e-05, "mean": 1.3058895916628899e-05, "normalized": [ 0.8914048885899613, 1.0775847922014457, 0.9537205719054163, 0.8574486865386339, 0.9048781790315702, 0.8425943237574262, 0.7839480584125434, 0.7118847066289794, 0.5788136508075115, 0.5307043127450849, 0.5023599445771596, 0.4844905191035793, 0.6659490427095206, 0.6212508418929359, 0.5836098650152179, 0.5530587758649329, 0.6705037956060123, 0.6390296236121042, 0.6539787044155942, 0.6823628402237571, 3.070552131396758, 2.5957343072679153, 2.1136920052807127, 1.723798777803649, 1.977372713282607, 1.75094231645673, 1.356176403898088, 0.9354566389044705, 0.8232143895880623, 0.5856320599681389, 0.4520406818345609, 0.4258114506789209 ], "peak_layer": 21, "peak_normalized": 3.070552131396758, "population_cv": 0.6442860381062553, "rest_mean": 9.924204715827256e-06, "spike_contrast": 3.0215245714485643, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.998622840095777e-05, "top_five_layers": [ 21, 22, 23, 25, 26 ], "top_five_spike_overlap": 4 }, "values": [ 1.1640763659670483e-05, 1.4072067642700858e-05, 1.2454537682060618e-05, 1.119733315135818e-05, 1.1816709957201965e-05, 1.100335157389054e-05, 1.0237496098852716e-05, 9.296428288507741e-06, 7.558667221019277e-06, 6.9304123826441355e-06, 6.560266228916589e-06, 6.3269112615671474e-06, 8.696559234522283e-06, 8.112850082397927e-06, 7.621300483151572e-06, 7.22233698979835e-06, 8.75603927852353e-06, 8.345021342393011e-06, 8.540239832655061e-06, 8.91090530785732e-06, 4.0098020690493286e-05, 3.389742414583452e-05, 2.7602483896771446e-05, 2.2510908820549957e-05, 2.5822304451139644e-05, 2.2865373466629535e-05, 1.7710166503093205e-05, 1.2216030881972983e-05, 1.0750271030701697e-05, 7.6477081165649e-06, 5.9031522141594905e-06, 5.560627414524788e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 4.568271015159553e-05, "first_to_last_ratio": 0.8467191676221869, "imbalance_abs_log_ratio": 0.16638620055756023, "last_quartile_mean": 5.3952611324348254e-05, "mean": 5.204496915212076e-05, "normalized": [ 0.8954358371924881, 1.066255627359817, 0.9521818394578381, 0.863653814778354, 0.9057514302498563, 0.8375363338827018, 0.7860525947143857, 0.7151691796362102, 0.5850370033455559, 0.5346182010583443, 0.5035057922216662, 0.4847392766839134, 0.6665845013282112, 0.6225484700608501, 0.5847766932080282, 0.5518779375078734, 0.6703635418666527, 0.6385663510046552, 0.6481965678809122, 0.6867100259137117, 3.061813692446919, 2.6065600609918396, 2.1204217920704185, 1.7184136066569389, 1.970772681476859, 1.7371423758267366, 1.3426072722615594, 0.9360215708260244, 0.8279779050062409, 0.5898318056357381, 0.4571555590952484, 0.4317206583534514 ], "peak_layer": 21, "peak_normalized": 3.061813692446919, "population_cv": 0.6425899958232327, "rest_mean": 3.9558066755826415e-05, "spike_contrast": 3.0202244925053914, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00011947424209211022, "top_five_layers": [ 21, 22, 23, 25, 26 ], "top_five_spike_overlap": 4 }, "values": [ 4.6602930524386466e-05, 5.549324123421684e-05, 4.955627446179278e-05, 4.494883614825085e-05, 4.7139805246843025e-05, 4.358955266070552e-05, 4.0910083043854684e-05, 3.722095789271407e-05, 3.044823279196862e-05, 2.7824187782243825e-05, 2.6204943424090743e-05, 2.5228240701835603e-05, 3.469236980890855e-05, 3.2400515920016915e-05, 3.0434684958891012e-05, 2.8722470233333297e-05, 3.488904985715635e-05, 3.3234166039619595e-05, 3.373537037987262e-05, 3.573980211513117e-05, 0.00015935199917294085, 0.0001356583379674703, 0.00011035728675778955, 8.943478314904496e-05, 0.00010256880341330543, 9.040952136274427e-05, 6.987595406826586e-05, 4.8715213779360056e-05, 4.309208452468738e-05, 3.0697778129251674e-05, 2.379264697083272e-05, 2.2468888346338645e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.04051188472658396, "first_to_last_ratio": 0.846719171620572, "imbalance_abs_log_ratio": 0.16638619583535103, "last_quartile_mean": 0.047845715656876564, "mean": 0.04615399963222444, "normalized": [ 0.8954358499233408, 1.0662556745533454, 0.9521818348722626, 0.8636537656053828, 0.9057513866007281, 0.8375363761309916, 0.7860525635386049, 0.7151692172158937, 0.5850370145184794, 0.53461818747956, 0.5035057498056084, 0.48473925875942064, 0.666584502816651, 0.6225484458781422, 0.5847766703673021, 0.551877902951673, 0.670363548905047, 0.6385663306455772, 0.6481965620941408, 0.6867099814534051, 3.0618138122732117, 2.606560153011009, 2.1204217916267223, 1.7184136164632142, 1.9707726965272079, 1.7371424139057434, 1.3426073387034982, 0.9360215296259088, 0.8279778593866045, 0.5898318104039583, 0.45715553160267397, 0.43172062235469105 ], "peak_layer": 21, "peak_normalized": 3.0618138122732117, "population_cv": 0.6425900249685824, "rest_mean": 0.03508048918512133, "spike_contrast": 3.020224589442675, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.10595095604658127, "top_five_layers": [ 21, 22, 23, 25, 26 ], "top_five_spike_overlap": 4 }, "values": [ 0.04132794588804245, 0.04921196401119232, 0.0439470000565052, 0.03986107558012009, 0.04180404916405678, 0.03865565359592438, 0.03627946972846985, 0.033007919788360596, 0.027001798152923584, 0.02467476762831211, 0.023238804191350937, 0.02237265557050705, 0.030765540897846222, 0.02873310074210167, 0.026989782229065895, 0.02547137252986431, 0.03093995898962021, 0.029472390189766884, 0.02991686388850212, 0.03169441223144531, 0.14131495356559753, 0.12030317634344101, 0.09786594659090042, 0.07931166142225266, 0.09095904231071472, 0.08017607033252716, 0.06196669861674309, 0.04320113733410835, 0.038214489817619324, 0.027223097160458565, 0.021099556237459183, 0.019925633445382118 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 1.8178741925112263e-06, "first_to_last_ratio": 0.5015983374707057, "imbalance_abs_log_ratio": 0.68995560412126, "last_quartile_mean": 3.6241631136135766e-06, "mean": 2.789416313930815e-06, "normalized": [ 0.8037911571270303, 0.7184217017994549, 0.6913125177981712, 0.6806867284307945, 0.605325927592527, 0.5818942838314957, 0.5658827679202331, 0.5663181288215361, 0.4369798716346615, 0.41689441779300684, 0.4078213694495809, 0.41540374891403675, 0.5056054983122913, 0.5034202979673874, 0.5059732849415142, 0.5032640783142611, 0.5409407291356678, 0.5584123873096231, 0.5703010167375573, 0.6038059140356934, 2.8715612743645713, 2.85957495161325, 2.4960596032794165, 2.196308283631167, 2.3935907596217825, 2.1483375605717328, 1.7201321959397329, 1.2992909597305788, 1.010652795641223, 0.7253891064801847, 0.563940476297644, 0.5327062049621921 ], "peak_layer": 21, "peak_normalized": 2.8715612743645713, "population_cv": 0.7722499515786573, "rest_mean": 1.981818833748413e-06, "spike_contrast": 3.608020362482595, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 7.150442706915783e-06, "top_five_layers": [ 21, 22, 23, 25, 24 ], "top_five_spike_overlap": 5 }, "values": [ 2.2421081666834652e-06, 2.0039772152813384e-06, 1.9283584151708055e-06, 1.8987186649610521e-06, 1.688506017671898e-06, 1.623145408302662e-06, 1.5784826246090233e-06, 1.5796970274095656e-06, 1.2189187827971182e-06, 1.1628920901785023e-06, 1.137583581112267e-06, 1.1587339940888342e-06, 1.4103442254054244e-06, 1.4042487919141422e-06, 1.4113701354290242e-06, 1.4038130302651552e-06, 1.508908894720662e-06, 1.5576446230625152e-06, 1.5908069599390728e-06, 1.6842660670590703e-06, 8.009979865164496e-06, 7.97654502093792e-06, 6.962549377931282e-06, 6.126418156782165e-06, 6.676721113763051e-06, 5.992607839289121e-06, 4.798164809471928e-06, 3.6242633996153018e-06, 2.8191313958814135e-06, 2.023412207563524e-06, 1.573064764670562e-06, 1.4859393786537112e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 3.4892004123321385e-05, "first_to_last_ratio": 1.010196136459353, "imbalance_abs_log_ratio": 0.010144506514065088, "last_quartile_mean": 3.453983129020344e-05, "mean": 3.719143947478187e-05, "normalized": [ 0.9787509726025058, 1.1268290755091765, 1.0180314063323372, 0.9373900555049763, 0.9573129653471193, 0.8927633987058169, 0.8376905027658148, 0.756615575252793, 0.6421721653954596, 0.5898966583006985, 0.5545320603024807, 0.5324801984524263, 0.7191238348729887, 0.6685886503056807, 0.6276517726739922, 0.5907083001617862, 0.7404052849968692, 0.7009964656156153, 0.7076026287863885, 0.7352288733491559, 3.079466745153095, 2.56966898996169, 2.031234533994799, 1.57522845930543, 1.8426476034454873, 1.5558698594237106, 1.200138700827154, 0.8699239581339129, 0.6815952172432139, 0.48817942111819174, 0.40443641967892363, 0.3868392464803102 ], "peak_layer": 21, "peak_normalized": 3.079466745153095, "population_cv": 0.6121837817284164, "rest_mean": 2.879134468390408e-05, "spike_contrast": 2.8672488989954323, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 8.255195134552196e-05, "top_five_layers": [ 21, 22, 23, 25, 24 ], "top_five_spike_overlap": 5 }, "values": [ 3.6401157558429986e-05, 4.190839536022395e-05, 3.786205343203619e-05, 3.486288551357575e-05, 3.5603847209131345e-05, 3.3203155908267945e-05, 3.11549156322144e-05, 2.813962237269152e-05, 2.388330722169485e-05, 2.193910586356651e-05, 2.06238455575658e-05, 1.9803705072263256e-05, 2.6745250579551794e-05, 2.4865774321369827e-05, 2.334327291464433e-05, 2.1969291992718354e-05, 2.7536738343769684e-05, 2.607106762297917e-05, 2.6316760340705514e-05, 2.7344220143277198e-05, 0.00011452980106696486, 9.556968871038407e-05, 7.554453623015434e-05, 5.85850139032118e-05, 6.853071681689471e-05, 5.786503970739432e-05, 4.463488585315645e-05, 3.23537242366001e-05, 2.5349507268401794e-05, 1.815609539335128e-05, 1.5041572623886168e-05, 1.4387108421942685e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 2.6184012767771492e-05, "first_to_last_ratio": 1.1812258678199312, "imbalance_abs_log_ratio": 0.1665527702583449, "last_quartile_mean": 2.2166812868817942e-05, "mean": 2.6416474952384306e-05, "normalized": [ 1.0583395321083173, 1.18417524025499, 1.0887520391639638, 1.0135132254608268, 1.0023744568550392, 0.9356357677270415, 0.8663869238627898, 0.7804236609988654, 0.6948391179201333, 0.645410883875558, 0.609013089334734, 0.5813567671594309, 0.7649466228246298, 0.711734220686414, 0.6645112375219416, 0.6291608441906736, 0.8117379342921259, 0.7682870001881678, 0.7656778963007645, 0.7856293365916892, 3.109981047613895, 2.532125441581352, 1.911742908862577, 1.3712180563471432, 1.6504233650379605, 1.361126344660198, 1.0867081917704957, 0.8610084831897756, 0.5742988763844462, 0.4302061202657988, 0.3792452416796221, 0.3700101252886388 ], "peak_layer": 21, "peak_normalized": 3.109981047613895, "population_cv": 0.5830394716350688, "rest_mean": 2.0961481856863463e-05, "spike_contrast": 2.6655289950267567, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.5873437668196856e-05, "top_five_layers": [ 21, 22, 23, 25, 24 ], "top_five_spike_overlap": 5 }, "values": [ 2.7957599741057493e-05, 3.1281735573429614e-05, 2.8760990971932188e-05, 2.6773446734296158e-05, 2.6479199732420966e-05, 2.471619882271625e-05, 2.2886888473294675e-05, 2.061604209302459e-05, 1.8355200154474005e-05, 1.7049480447894894e-05, 1.6087979020085186e-05, 1.535739647806622e-05, 2.0207193301757798e-05, 1.880150921351742e-05, 1.7554044461576268e-05, 1.6620211681583896e-05, 2.144325480912812e-05, 2.029543429671321e-05, 2.0226510969223455e-05, 2.0753557691932656e-05, 8.215473644668236e-05, 6.688982830382884e-05, 5.050150866736658e-05, 3.62227474397514e-05, 4.3598367483355105e-05, 3.595615999074653e-05, 2.870699972845614e-05, 2.2744809029973112e-05, 1.5170951883192174e-05, 1.1364529200363904e-05, 1.0018322427640669e-05, 9.774363206815906e-06 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 9.068014514923561e-05, "first_to_last_ratio": 0.7913424091198598, "imbalance_abs_log_ratio": 0.23402452356595546, "last_quartile_mean": 0.00011459027609816985, "mean": 0.00010754564084436424, "normalized": [ 0.8540109519123018, 1.0092398375236236, 0.8947367225413311, 0.8160956211381821, 0.862539829414268, 0.8245016378814637, 0.7800932919128317, 0.7042081007547356, 0.5666342926354664, 0.5256616841881218, 0.49571278437559146, 0.4792103383514405, 0.6722886591537548, 0.6295888625242309, 0.5934294393111791, 0.5594389550092885, 0.6645012760984752, 0.6282095204823469, 0.6412757892302924, 0.6736811938431846, 3.0068371075614557, 2.616557188534586, 2.1686834473810053, 1.8088340274566967, 2.0615653735311414, 1.8302172118360363, 1.4236627074872454, 0.9903997896635313, 0.8090570392266916, 0.5559293048344423, 0.43884492996422475, 0.4143530842408338 ], "peak_layer": 21, "peak_normalized": 3.0068371075614557, "population_cv": 0.6573647361404942, "rest_mean": 8.100784921057143e-05, "spike_contrast": 3.0966099965793172, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00025084971566684543, "top_five_layers": [ 21, 22, 23, 25, 26 ], "top_five_spike_overlap": 4 }, "values": [ 9.184515511151403e-05, 0.00010853934509214014, 9.622503421269357e-05, 8.776752656558529e-05, 9.276239870814607e-05, 8.867155702318996e-05, 8.389563299715519e-05, 7.573451148346066e-05, 6.093904812587425e-05, 5.653262269333936e-05, 5.3311749070417136e-05, 5.1536982937250286e-05, 7.230171468108892e-05, 6.770953768864274e-05, 6.382074934663251e-05, 6.016522092977539e-05, 7.146421557990834e-05, 6.756119546480477e-05, 6.896641571074724e-05, 7.245147571666166e-05, 0.00032337222364731133, 0.00028139931964688003, 0.0002332324511371553, 0.0001945322146639228, 0.00022171236923895776, 0.00019683188293129206, 0.00015310871822293848, 0.00010651318007148802, 8.701055776327848e-05, 5.978777335258201e-05, 4.7195859224302694e-05, 4.456186798051931e-05 ] } } }, "post_mlp_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "89a6b034a7ebb02fe6279d25c4264a8a3e749e8c9a49e8aa75fc34af7ffbb3bf", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.16101159155368805, 0.3334391713142395, 0.3570481836795807, 0.37708213925361633, 0.11129721254110336, 0.12943366169929504, 0.1512180119752884, 0.1945808082818985, 0.07549218833446503, 0.11765842139720917, 0.16134388744831085, 0.21493762731552124, 0.08329613506793976, 0.12312135100364685, 0.16809259355068207, 0.23101076483726501, 0.11538279056549072, 0.16158254444599152, 0.20080390572547913, 0.27125677466392517, 0.0543946772813797, 0.09335283935070038, 0.12897421419620514, 0.17114612460136414, 0.06173322722315788, 0.09693212062120438, 0.13771499693393707, 0.18387214839458466, 0.07983565330505371, 0.15526357293128967, 0.26127612590789795, 0.39550676941871643 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.1373772622391698e-05, "first_to_last_ratio": 1.0318485034659548, "imbalance_abs_log_ratio": 0.03135185731624994, "last_quartile_mean": 1.1022715625585988e-05, "mean": 1.1832133452571725e-05, "normalized": [ 0.9800044623886516, 1.1858619463319722, 1.052769444074511, 0.9435868051213633, 0.9942662730454839, 0.9213689272443336, 0.8458140758926892, 0.7664188972680795, 0.6009172873544569, 0.5652322062780974, 0.5379371038219452, 0.5220047919249511, 0.7166699752090641, 0.6623360432121947, 0.616224251829878, 0.5903966461838739, 0.7324373700274538, 0.6882103836910076, 0.7092295284797906, 0.8089552049077647, 3.1154122124911208, 2.432133583525468, 1.964217211992137, 1.5948629117896413, 1.9522993684268721, 1.5038120812358047, 1.036848162520458, 0.8807155414212365, 0.6474761343085111, 0.5016554233955102, 0.4711561792646531, 0.4587695653410252 ], "peak_layer": 21, "peak_normalized": 3.1154122124911208, "population_cv": 0.6032722735217898, "rest_mean": 9.176947801110887e-06, "spike_contrast": 2.851725490613808, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.617013597046025e-05, "top_five_layers": [ 21, 22, 23, 25, 24 ], "top_five_spike_overlap": 5 }, "values": [ 1.1595543583098333e-05, 1.4031276805326343e-05, 1.245650855707936e-05, 1.1164645002281759e-05, 1.1764291230065282e-05, 1.0901760106207803e-05, 1.0007785022025928e-05, 9.068370673048776e-06, 7.110133537935326e-06, 6.687902896373998e-06, 6.364943601511186e-06, 6.176430360937957e-06, 8.479734788124915e-06, 7.836848453735001e-06, 7.2912475843622815e-06, 6.985651907598367e-06, 8.66629670781549e-06, 8.142997103277594e-06, 8.391698429477401e-06, 9.571665941621177e-06, 3.686197305796668e-05, 2.8777329134754837e-05, 2.324088018212933e-05, 1.8870630810852163e-05, 2.3099866666598246e-05, 1.7793305232771672e-05, 1.2268125828995835e-05, 1.0420743819850031e-05, 7.661024028493557e-06, 5.935653916822048e-06, 5.574782790063182e-06, 5.428222721093334e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 4.529294619715074e-05, "first_to_last_ratio": 1.0314184626645753, "imbalance_abs_log_ratio": 0.03093500306067748, "last_quartile_mean": 4.391325910546584e-05, "mean": 4.717383586694268e-05, "normalized": [ 0.9832121149293677, 1.1731445898725168, 1.0505704052806353, 0.9501685038967237, 0.9944626347672784, 0.9147662213723499, 0.8464650022543867, 0.7682388544141363, 0.6067490194436969, 0.5681705700165066, 0.53864641920515, 0.5211305553096096, 0.7167263481864653, 0.6632364446817158, 0.6172546895643319, 0.588531029519173, 0.7318735922946267, 0.6893643707481226, 0.7026758072754231, 0.8079900073900919, 3.115049509935762, 2.4538624912763685, 1.9660898437352061, 1.5845676157597377, 1.937541165727475, 1.4892951790023068, 1.0350080319051866, 0.8840334683174624, 0.6516976556256888, 0.50650640646803, 0.47764736477161285, 0.4653240870528551 ], "peak_layer": 21, "peak_normalized": 3.115049509935762, "population_cv": 0.6027314599912318, "rest_mean": 3.6590978736596835e-05, "spike_contrast": 2.8510104941924466, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00010432126437081024, "top_five_layers": [ 21, 22, 23, 25, 24 ], "top_five_spike_overlap": 5 }, "values": [ 4.638188693206757e-05, 5.534173033083789e-05, 4.9559435865376145e-05, 4.482309304876253e-05, 4.6912617108318955e-05, 4.315303158364259e-05, 3.9931001083459705e-05, 3.624077362474054e-05, 2.8622678655665368e-05, 2.680278521438595e-05, 2.541001776990015e-05, 2.4583727281424217e-05, 3.3810731110861525e-05, 3.128740718238987e-05, 2.9118271413608454e-05, 2.7763266189140268e-05, 3.4525284718256444e-05, 3.251996167819016e-05, 3.314791320008226e-05, 3.811598799075e-05, 0.00014694883429910988, 0.00011575810640351847, 9.27479995880276e-05, 7.475013262592256e-05, 9.14012489374727e-05, 7.025576633168384e-05, 4.882529901806265e-05, 4.170324973529205e-05, 3.074307824135758e-05, 2.38938500842778e-05, 2.2532458388013765e-05, 2.1951122107566334e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.040166237857192755, "first_to_last_ratio": 1.0314184457192797, "imbalance_abs_log_ratio": 0.030934986631559365, "last_quartile_mean": 0.038942718179896474, "mean": 0.0418342292541638, "normalized": [ 0.9832121171072725, 1.1731446022953238, 1.0505704535174525, 0.9501684905945749, 0.9944626387492155, 0.9147662270065798, 0.8464650412985084, 0.7682388261849044, 0.6067490356906232, 0.5681705774712352, 0.5386464289336567, 0.5211305632866429, 0.7167263331576038, 0.6632364212118128, 0.6172547081682093, 0.5885310198061827, 0.7318735882573365, 0.6893643353923572, 0.7026757592562258, 0.8079899688020947, 3.1150494118332737, 2.453862539114119, 1.9660897667076618, 1.5845675971028321, 1.9375412374473495, 1.4892952935873054, 1.0350080081219681, 0.8840335073818288, 0.6516976484197683, 0.5065064256973429, 0.4776473385482515, 0.46532408985048596 ], "peak_layer": 21, "peak_normalized": 3.1150494118332737, "population_cv": 0.6027314563638121, "rest_mean": 0.03244924586680201, "spike_contrast": 2.851010464947826, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.09251313954591751, "top_five_layers": [ 21, 22, 23, 25, 24 ], "top_five_spike_overlap": 5 }, "values": [ 0.041131921112537384, 0.0490776002407074, 0.043949805200099945, 0.039749566465616226, 0.04160257801413536, 0.03826854005455971, 0.03541121259331703, 0.032138679176568985, 0.02538287825882435, 0.02376897819340229, 0.022533858194947243, 0.021801095455884933, 0.02998369373381138, 0.027745984494686127, 0.02582237496972084, 0.024620741605758667, 0.0306173674762249, 0.028839025646448135, 0.029395898804068565, 0.03380163758993149, 0.13031569123268127, 0.10265544801950455, 0.08224985003471375, 0.06628916412591934, 0.08105554431676865, 0.06230352073907852, 0.043298762291669846, 0.036982860416173935, 0.02726326882839203, 0.021189305931329727, 0.019982008263468742, 0.019466474652290344 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 1.8091637201678168e-06, "first_to_last_ratio": 0.5997838633923696, "imbalance_abs_log_ratio": 0.511185916342951, "last_quartile_mean": 3.0163594431087404e-06, "mean": 2.4871596693287756e-06, "normalized": [ 0.8903650291088842, 0.8096671832364504, 0.7756357252979299, 0.7664205976400669, 0.6819345308185697, 0.6497344242040057, 0.6267720257428505, 0.6186826317778965, 0.4671058445444584, 0.45225062104576175, 0.4492165155127133, 0.4573049495780591, 0.555081192151318, 0.5467853740610928, 0.5411086649066317, 0.5362598920190076, 0.601197329062199, 0.609347333830986, 0.6241580819811688, 0.6880678322200046, 2.9157464374333077, 2.626463595143142, 2.355941396471949, 2.0525708846936164, 2.4216180933419804, 1.9231831825131547, 1.4564932487560838, 1.2844496192048258, 0.8068213094456221, 0.6295223209584038, 0.59698327836313, 0.5831108549347289 ], "peak_layer": 21, "peak_normalized": 2.9157464374333077, "population_cv": 0.7105915967606766, "rest_mean": 1.8080416052894347e-06, "spike_contrast": 3.4039024307496555, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.154397215141216e-06, "top_five_layers": [ 21, 22, 25, 23, 24 ], "top_five_spike_overlap": 5 }, "values": [ 2.214479991380358e-06, 2.013771563724731e-06, 1.9291298940515844e-06, 1.9062104001932312e-06, 1.6960800621745875e-06, 1.6159932556547574e-06, 1.558882104291115e-06, 1.5387624898721697e-06, 1.1617668178587337e-06, 1.1248195050939103e-06, 1.1172732001796248e-06, 1.137390427174978e-06, 1.3805755543216947e-06, 1.3599425301435986e-06, 1.3458236480801133e-06, 1.3337639757082798e-06, 1.4952737501516822e-06, 1.5155441133174463e-06, 1.5523808087891666e-06, 1.711334562060074e-06, 7.251926945173182e-06, 6.532434326800285e-06, 5.8596024246071465e-06, 5.105071522848448e-06, 6.022950856277021e-06, 4.78326364827808e-06, 3.622531266955775e-06, 3.194631290170946e-06, 2.0066934212081833e-06, 1.5657225276299869e-06, 1.484792733208451e-06, 1.45028980114148e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 3.464059318503132e-05, "first_to_last_ratio": 1.2085006098527633, "imbalance_abs_log_ratio": 0.1893804257974035, "last_quartile_mean": 2.8664108981502068e-05, "mean": 3.421737719122575e-05, "normalized": [ 1.0599522534665642, 1.2224211904533802, 1.1066526183773722, 1.0166167984558396, 1.0363163370114612, 0.9614978327769692, 0.8912999264066335, 0.8041906605597857, 0.6595162801975746, 0.6198396025381588, 0.5858589457780455, 0.5655951157979022, 0.7633864100877904, 0.7040498510939207, 0.65484714742333, 0.6226490455428274, 0.7974248515656103, 0.7481679248848441, 0.7573152935416273, 0.844065677484853, 3.1218032666740525, 2.4268896738358103, 1.8727607046170862, 1.4552330006230305, 1.7605509694422596, 1.3320662835665362, 0.9562601026332771, 0.8300407364235332, 0.5336839174330044, 0.4444323110860877, 0.4233154787321452, 0.421299791488687 ], "peak_layer": 21, "peak_normalized": 3.1218032666740525, "population_cv": 0.5796332934591588, "rest_mean": 2.7073248087685056e-05, "spike_contrast": 2.6888415499527167, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 7.279567435034551e-05, "top_five_layers": [ 21, 22, 23, 25, 24 ], "top_five_spike_overlap": 5 }, "values": [ 3.626878606155515e-05, 4.182804696029052e-05, 3.786675006267615e-05, 3.478596045169979e-05, 3.546002699295059e-05, 3.289993401267566e-05, 3.049794577236753e-05, 2.7517295166035183e-05, 2.2566917323274538e-05, 2.120928547810763e-05, 2.004655652854126e-05, 1.9353181414771825e-05, 2.6121080736629665e-05, 2.4090739316307008e-05, 2.2407151845982298e-05, 2.1305417249095626e-05, 2.728578692767769e-05, 2.5600344088161364e-05, 2.591334305179771e-05, 2.888171366066672e-05, 0.00010681991989258677, 8.304179937113076e-05, 6.408095941878855e-05, 4.9794256483437493e-05, 6.024143658578396e-05, 4.557981446851045e-05, 3.272071262472309e-05, 2.840181696228683e-05, 1.826126390369609e-05, 1.5207308024400845e-05, 1.4484745406662114e-05, 1.4415773875953164e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 2.6039962676804862e-05, "first_to_last_ratio": 1.3600902622782072, "imbalance_abs_log_ratio": 0.3075510668678258, "last_quartile_mean": 1.914576068884344e-05, "mean": 2.476154466535263e-05, "normalized": [ 1.123958441324931, 1.2626354632798173, 1.1622790571900958, 1.0807265666022514, 1.0670121245383055, 0.9899364488555507, 0.9055408259899391, 0.8209444362453999, 0.7058751897560187, 0.6697662418103106, 0.6333373009906147, 0.6058288961358885, 0.7980579670309706, 0.7373130204202144, 0.6858704929138739, 0.6566823825816065, 0.8578675442054349, 0.8063397293421843, 0.8071587377333899, 0.875107050101149, 3.1369738883948477, 2.3886515998281816, 1.7583385106457243, 1.2781546769066958, 1.5599158953991201, 1.1978164843461745, 0.9315129197863214, 0.8390098557963587, 0.46047650401210477, 0.40917670978266013, 0.3951608602894278, 0.39257417776443665 ], "peak_layer": 21, "peak_normalized": 3.1369738883948477, "population_cv": 0.558754889865697, "rest_mean": 2.0064156227885246e-05, "spike_contrast": 2.4983578506037145, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.012744222767651e-05, "top_five_layers": [ 21, 22, 23, 25, 24 ], "top_five_spike_overlap": 5 }, "values": [ 2.7830947146867402e-05, 3.126480442006141e-05, 2.87798247882165e-05, 2.676045914995484e-05, 2.6420868380228058e-05, 2.4512355594197288e-05, 2.2422589609050192e-05, 2.0327852325863205e-05, 1.7478560039307922e-05, 1.6584446711931378e-05, 1.5682409866712987e-05, 1.5001259271230083e-05, 1.9761147996177897e-05, 1.8257009287481196e-05, 1.6983212844934314e-05, 1.6260470147244632e-05, 2.1242125512799248e-05, 1.996621722355485e-05, 1.9986497136414982e-05, 2.1669002308044583e-05, 7.767631905153394e-05, 5.9146703279111534e-05, 4.353917756816372e-05, 3.164908412145451e-05, 3.862592711811885e-05, 2.965978637803346e-05, 2.306569876964204e-05, 2.0775180018972605e-05, 1.1402109521441162e-05, 1.013184737530537e-05, 9.784793292055838e-06, 9.720743037178181e-06 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 9.000182035379112e-05, "first_to_last_ratio": 0.9625877483741323, "imbalance_abs_log_ratio": 0.03813004983371345, "last_quartile_mean": 9.349986066808924e-05, "mean": 9.762683816916251e-05, "normalized": [ 0.9359774389006617, 1.1139547805652443, 0.9858638933617162, 0.896277058777463, 0.9496551618529471, 0.8993751236921529, 0.8464771316171845, 0.7475897673874267, 0.588915541149348, 0.5540229040457487, 0.5295015812507878, 0.5183640792456576, 0.7262152137199526, 0.6676039603614504, 0.6223324443316169, 0.5861986137162843, 0.7254235746253427, 0.6817432367794596, 0.6942948378413707, 0.7958728955594262, 3.0862176056110457, 2.4751904263802063, 2.0171676620749306, 1.6939489172879052, 2.0229993487511715, 1.572447147387479, 1.0908649591739474, 0.9643206077313163, 0.6147279541208738, 0.4857837184299343, 0.45630950884426186, 0.45436290542568586 ], "peak_layer": 21, "peak_normalized": 3.0862176056110457, "population_cv": 0.617326253069026, "rest_mean": 7.486342710089284e-05, "spike_contrast": 2.946021394940767, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00022054925793781876, "top_five_layers": [ 21, 22, 25, 23, 24 ], "top_five_spike_overlap": 5 }, "values": [ 9.137651795754209e-05, 0.00010875188309000805, 9.624677477404475e-05, 8.750069537200034e-05, 9.271183080272749e-05, 8.780314965406433e-05, 8.263888594228774e-05, 7.298482523765415e-05, 5.7493962231092155e-05, 5.408750439528376e-05, 5.169356518308632e-05, 5.060624607722275e-05, 7.089809514582157e-05, 6.51760637992993e-05, 6.0756348830182105e-05, 5.72287171962671e-05, 7.082080992404372e-05, 6.655643664998934e-05, 6.778180977562442e-05, 7.769855437800288e-05, 0.0003012976667378098, 0.00024164501519408077, 0.00019692970090545714, 0.00016537487681489438, 0.00019749903003685176, 0.00015351304318755865, 0.00010649769683368504, 9.414357191417366e-05, 6.00139464950189e-05, 4.74255284643732e-05, 4.454805457498878e-05, 4.4358013838063926e-05 ] } } }, "pre_attention_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "655dba72ac4e6dee3e5a00c08f2fe22c99d59ee39e0a5d692a12f5d8d0e85bda", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.059412047266960144, 0.08935670554637909, 0.17747226357460022, 0.17946027219295502, 0.17260213196277618, 0.12674333155155182, 0.12412907183170319, 0.11767174303531647, 0.13582420349121094, 0.10855290293693542, 0.10545514523983002, 0.11005345731973648, 0.13049271702766418, 0.09666567295789719, 0.10358717292547226, 0.11041814833879471, 0.13209649920463562, 0.10239993035793304, 0.11153820157051086, 0.11912636458873749, 0.127323716878891, 0.07468672841787338, 0.07975661009550095, 0.08746611326932907, 0.09389390796422958, 0.06575146317481995, 0.0681111291050911, 0.07568862289190292, 0.08424623310565948, 0.0719028189778328, 0.08439797908067703, 0.08961156010627747 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 7.588972493977053e-06, "first_to_last_ratio": 1.465723686175851, "imbalance_abs_log_ratio": 0.38234910423701196, "last_quartile_mean": 5.1776283385152055e-06, "mean": 8.926610817638903e-06, "normalized": [ 0.8001707694815087, 1.2301759322409134, 0.8021990606376289, 0.7996309276786352, 0.4189560575118035, 0.770181199703876, 0.9964730820564364, 0.983425792900568, 0.6138765442017692, 0.5043130598626571, 0.39700601441149197, 0.37391212513577055, 0.38881218172106297, 0.7038080142822467, 0.6284832461563316, 0.48736568495191596, 0.7748207724466236, 1.0896235823984757, 1.2330955711393075, 1.3976034073521157, 2.3882743999212943, 4.5696632038041, 2.815663441519544, 2.192292008694111, 1.06869084333539, 1.167599927936825, 0.6186038915435874, 0.511201557417111, 0.4679157405813128, 0.21996309432798952, 0.26258255269417496, 0.3236163119534213 ], "peak_layer": 22, "peak_normalized": 4.5696632038041, "population_cv": 0.8757958168839719, "rest_mean": 6.270255131244927e-06, "spike_contrast": 3.711321315811602, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.3270931524166372e-05, "top_five_layers": [ 22, 23, 21, 24, 20 ], "top_five_spike_overlap": 4 }, "values": [ 7.14281304681208e-06, 1.0981301784340758e-05, 7.160918812587624e-06, 7.137994089134736e-06, 3.739857675100211e-06, 6.875107828818727e-06, 8.895127393770963e-06, 8.778659321251325e-06, 5.479837000166299e-06, 4.50180641564657e-06, 3.5439181829133304e-06, 3.3377680210833205e-06, 3.4707750273810234e-06, 6.2826202338328585e-06, 5.610225343843922e-06, 4.350523795437766e-06, 6.916523489053361e-06, 9.726645657792687e-06, 1.1007364264514763e-05, 1.2475861694838386e-05, 2.1319196093827486e-05, 4.079160498804413e-05, 2.513433173589874e-05, 1.956973756023217e-05, 9.539787242829334e-06, 1.0422710147395264e-05, 5.52203619008651e-06, 4.5632973524334375e-06, 4.176901711616665e-06, 1.9635249373095576e-06, 2.3439722554030595e-06, 2.8887968710478162e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 2.9898558636887174e-05, "first_to_last_ratio": 1.4621096880940556, "imbalance_abs_log_ratio": 0.37988038456939854, "last_quartile_mean": 2.0448916302484577e-05, "mean": 3.561926479278554e-05, "normalized": [ 0.7085385219770569, 1.1946020815752356, 0.7754024698062456, 0.7950922364389753, 0.4162410374634061, 0.8241424849247687, 0.9939701500428968, 1.0071543674888268, 0.5806040892681411, 0.5114068129628899, 0.4225456612843993, 0.35601883603809836, 0.39596565343777096, 0.700750155357531, 0.608338530669077, 0.5057982656951122, 0.7834207937068051, 1.0599792777683275, 1.2280812674610306, 1.4767524648757513, 2.409304768407655, 4.508238395616571, 2.902287830939038, 2.2425871184827204, 1.0734427327031963, 1.1671335490266965, 0.5965647485206973, 0.46999739473988966, 0.46865462397650987, 0.22456393251833243, 0.25925607539704565, 0.3331636714293014 ], "peak_layer": 22, "peak_normalized": 4.508238395616571, "population_cv": 0.8793156123901573, "rest_mean": 2.4886176578109843e-05, "spike_contrast": 3.760237771291333, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.357794115203433e-05, "top_five_layers": [ 22, 23, 21, 24, 20 ], "top_five_spike_overlap": 4 }, "values": [ 2.523762123018969e-05, 4.255084786564112e-05, 2.761926589300856e-05, 2.832060090440791e-05, 1.482619973103283e-05, 2.9355349397519603e-05, 3.540448597050272e-05, 3.587409810279496e-05, 2.0680690795416012e-05, 1.8215934687759727e-05, 1.5050765796331689e-05, 1.2681129192060325e-05, 1.4104005458648317e-05, 2.4960205337265506e-05, 2.1668571207555942e-05, 1.80161623575259e-05, 2.790487269521691e-05, 3.775568256969564e-05, 4.374335185275413e-05, 5.2600837079808116e-05, 8.581766451243311e-05, 0.00016058013716246933, 0.00010337735875509679, 7.987930439412594e-05, 3.8235240936046466e-05, 4.157243893132545e-05, 2.1249197743600234e-05, 1.674096165515948e-05, 1.6693133147782646e-05, 7.998802175279707e-06, 9.234510798705742e-06, 1.1867045031976886e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.026514341006986797, "first_to_last_ratio": 1.4621096829473716, "imbalance_abs_log_ratio": 0.379880381049359, "last_quartile_mean": 0.01813430368201807, "mean": 0.03158751998853404, "normalized": [ 0.7085385035567081, 1.1946021626253802, 0.7754024540661398, 0.7950922593029921, 0.4162410309209195, 0.8241424880807221, 0.9939701540557254, 1.0071544006568143, 0.5806040665606002, 0.5114068209416861, 0.4225456864143744, 0.3560188250781175, 0.39596564639005655, 0.7007501603721394, 0.6083384888465566, 0.5057982797995526, 0.7834208304641355, 1.059979249016567, 1.2280813415682141, 1.4767524934129506, 2.409304781935525, 4.508238208880784, 2.902287717411148, 2.2425871343428208, 1.0734427669067585, 1.167133566980318, 0.5965647509522538, 0.46999742239503073, 0.4686546086713296, 0.22456393938088035, 0.25925609979218267, 0.3331636602206165 ], "peak_layer": 22, "peak_normalized": 4.508238208880784, "population_cv": 0.8793155841158318, "rest_mean": 0.022069310372764314, "spike_contrast": 3.7602376563657023, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.08298585191369057, "top_five_layers": [ 22, 23, 21, 24, 20 ], "top_five_spike_overlap": 4 }, "values": [ 0.022380974143743515, 0.03773451969027519, 0.024493040516972542, 0.025114992633461952, 0.013148021884262562, 0.026032617315649986, 0.031397052109241486, 0.03181350976228714, 0.018339842557907104, 0.016154073178768158, 0.013347170315682888, 0.01124575175344944, 0.012507572770118713, 0.02213495969772339, 0.0192159041762352, 0.015976913273334503, 0.024746321141719818, 0.03348211571574211, 0.038792043924331665, 0.04664694890379906, 0.07610396295785904, 0.14240406453609467, 0.09167607128620148, 0.07083776593208313, 0.03390739485621452, 0.03686685487627983, 0.01884400099515915, 0.01484605297446251, 0.014803636819124222, 0.007093417923897505, 0.008189257234334946, 0.010523813776671886 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 2.343415339112198e-06, "first_to_last_ratio": 0.7161840718970601, "imbalance_abs_log_ratio": 0.33381806140671977, "last_quartile_mean": 3.272085251637691e-06, "mean": 2.6718059498165303e-06, "normalized": [ 3.352170512510883, 1.252135055595148, 0.26582849245093404, 0.3779079423767161, 0.26494682358528443, 0.3763245935115975, 0.4665297795225812, 0.6608799701010516, 0.3129183029559072, 0.27411212897478776, 0.2550957946560799, 0.3550642916181046, 0.2737856172573793, 0.47262961496261435, 0.48685762845780345, 0.5118283391293013, 0.3284988072562945, 0.6155436681481607, 0.5758322948420541, 0.7857703722429643, 1.46149784943323, 3.1539420937482103, 2.83862849215308, 2.483897584167724, 1.672423695519606, 1.9844503515036886, 1.6695842115910071, 1.4607872550915708, 1.1268636557445397, 0.7047297682619745, 0.577264376468574, 0.6012706361611467 ], "peak_layer": 1, "peak_normalized": 3.352170512510883, "population_cv": 0.8823003320995952, "rest_mean": 2.0176697064387014e-06, "spike_contrast": 3.074904502088927, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.204141664056806e-06, "top_five_layers": [ 1, 22, 23, 24, 26 ], "top_five_spike_overlap": 3 }, "values": [ 8.956349120126106e-06, 3.3454618915129686e-06, 7.102421477611642e-07, 1.0096966889250325e-06, 7.078864996401535e-07, 1.0054662880065734e-06, 1.2464770406950265e-06, 1.7657430362305604e-06, 8.360569836440845e-07, 7.323744171117141e-07, 6.815664619352901e-07, 9.486628869126434e-07, 7.315020411624573e-07, 1.2627746173166088e-06, 1.300789108427125e-06, 1.36750600177038e-06, 8.776850677350012e-07, 1.6446132349301479e-06, 1.5385121514555067e-06, 2.099425955748302e-06, 3.904838649759768e-06, 8.426721251453273e-06, 7.5842644946533255e-06, 6.636492344114231e-06, 4.468391580303432e-06, 5.3020662562630605e-06, 4.4608050302485935e-06, 3.902940079569817e-06, 3.0107610200502677e-06, 1.8829011878551682e-06, 1.5423383956658654e-06, 1.6064784631453222e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 1.9795748357864795e-05, "first_to_last_ratio": 1.5829781482320753, "imbalance_abs_log_ratio": 0.4593079767809046, "last_quartile_mean": 1.2505383210736909e-05, "mean": 2.345933930314459e-05, "normalized": [ 0.6329672888423138, 1.1769775978635326, 0.8122053642936898, 0.8103566266208889, 0.46133266366701603, 0.8656603467447103, 1.0301469021057994, 0.9610116498331323, 0.61408958600413, 0.5510707221672605, 0.4548082316962977, 0.3885266091083219, 0.4407469545604743, 0.7576792724096253, 0.6251552614433971, 0.5326228510318479, 0.8273444966654151, 1.114509292339196, 1.2367312992724153, 1.4148882628934067, 2.3782100012844283, 4.280445457492481, 3.0878808679778498, 2.2801019909661657, 1.0813850741694784, 1.1060747126282833, 0.5844412396792442, 0.4054991266359997, 0.4197995696214123, 0.21544640490341943, 0.19403464509867754, 0.2578496299796893 ], "peak_layer": 22, "peak_normalized": 4.280445457492481, "population_cv": 0.8646936695537649, "rest_mean": 1.6414566272470876e-05, "spike_contrast": 3.746740099488776, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.150111366878263e-05, "top_five_layers": [ 22, 23, 21, 24, 20 ], "top_five_spike_overlap": 4 }, "values": [ 1.4848994396743365e-05, 2.7611116820480675e-05, 1.9053801224799827e-05, 1.9010431060451083e-05, 1.0822559488588013e-05, 2.0307819795561954e-05, 2.416656570858322e-05, 2.2544698367710225e-05, 1.4406135960598476e-05, 1.2927755051350687e-05, 1.0669500625226647e-05, 9.11457755137235e-06, 1.0339632353861816e-05, 1.7774655134417117e-05, 1.4665729395346716e-05, 1.2494980182964355e-05, 1.9408955267863348e-05, 2.6145651645492762e-05, 2.9012899176450446e-05, 3.319234383525327e-05, 5.5791235354263335e-05, 0.00010041642235592008, 7.2439645009581e-05, 5.34896862518508e-05, 2.5368579372297972e-05, 2.5947781978175044e-05, 1.371060534438584e-05, 9.512741598882712e-06, 9.84822054306278e-06, 5.05423031427199e-06, 4.551924575935118e-06, 6.0489819588838145e-06 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 1.3026621559220075e-05, "first_to_last_ratio": 1.6815341811376112, "imbalance_abs_log_ratio": 0.5197065797570352, "last_quartile_mean": 7.74686694171578e-06, "mean": 1.525919709166601e-05, "normalized": [ 0.5774672585272599, 1.1989902313733303, 0.8415468606033967, 0.8210487753497788, 0.5264205831559432, 0.9142648476833113, 1.0733362714129566, 0.8764439105095523, 0.6569979339342769, 0.6167902521881127, 0.4964350034575567, 0.41696619601743, 0.504954663560933, 0.8317766688879529, 0.6542358689500138, 0.5551252336862755, 0.852113468285689, 1.140364193630489, 1.2382881917527824, 1.3141660941820785, 2.300727833390314, 4.0533116788495525, 3.2452755011871197, 2.231471710715914, 1.0423588969466941, 1.097025384808554, 0.5832116217690613, 0.3693409741830814, 0.3982639506890586, 0.21137018039235425, 0.16048903759429983, 0.19942072232487634 ], "peak_layer": 22, "peak_normalized": 4.0533116788495525, "population_cv": 0.8414682188802379, "rest_mean": 1.0809645952273664e-05, "spike_contrast": 3.6344181315319797, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.9286773244384676e-05, "top_five_layers": [ 22, 23, 21, 24, 20 ], "top_five_spike_overlap": 4 }, "values": [ 8.811686711851507e-06, 1.8295628251507878e-05, 1.2841329407820012e-05, 1.2528545084933285e-05, 8.032755431486294e-06, 1.3950947504781652e-05, 1.6378249711124226e-05, 1.3373830370255746e-05, 1.0025260962720495e-05, 9.411724022356793e-06, 7.575199560960755e-06, 6.362569365592208e-06, 7.705202733632177e-06, 1.2692244126810692e-05, 9.983114068745635e-06, 8.47076535137603e-06, 1.3002567357034422e-05, 1.7401041986886412e-05, 1.889528357423842e-05, 2.0053119442309253e-05, 3.510725946398452e-05, 6.185028178151697e-05, 4.952029848936945e-05, 3.4050466638291255e-05, 1.5905559848761186e-05, 1.6739726561354473e-05, 8.899341082724277e-06, 5.635846719087567e-06, 6.077188118069898e-06, 3.225339241907932e-06, 2.448933855703217e-06, 3.043000106117688e-06 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 5.830217355651257e-05, "first_to_last_ratio": 1.5342432776570591, "imbalance_abs_log_ratio": 0.42803728076015524, "last_quartile_mean": 3.8000605513843766e-05, "mean": 7.032809671159157e-05, "normalized": [ 0.6819589535091998, 1.1712319271696168, 0.7869167841712673, 0.779993936454377, 0.40965657804504296, 0.8112730725890709, 0.9714853978881673, 1.0195039829060222, 0.5575391424076114, 0.48579339303929814, 0.4244288891192081, 0.3806909807485322, 0.3969007539297048, 0.6868963514684149, 0.5698714639132343, 0.5145191503639089, 0.8281057882616928, 1.0694789461698069, 1.2364533982124963, 1.5208781026521505, 2.5059295507917216, 4.611853308457518, 2.9277607037675337, 2.3282138476317833, 1.1223656127309714, 1.0704287879823866, 0.5800732884744664, 0.412100266213389, 0.4197925520111647, 0.21365215237778729, 0.20903494194495364, 0.2952179945975007 ], "peak_layer": 22, "peak_normalized": 4.611853308457518, "population_cv": 0.9090106345148538, "rest_mean": 4.819786850189471e-05, "spike_contrast": 3.9385834881161212, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001898313290439546, "top_five_layers": [ 22, 23, 21, 24, 20 ], "top_five_spike_overlap": 4 }, "values": [ 4.796087523573078e-05, 8.237051224568859e-05, 5.534235970117152e-05, 5.4855488997418433e-05, 2.881036743929144e-05, 5.705529110855423e-05, 6.832271901657805e-05, 7.169977470766753e-05, 3.921066672774032e-05, 3.4164924727519974e-05, 2.984927596116904e-05, 2.6773272111313418e-05, 2.7913274607271887e-05, 4.830811303691007e-05, 4.007797542726621e-05, 3.6185152566758916e-05, 5.82391039642971e-05, 7.521441875724122e-05, 8.695741416886449e-05, 0.00010696046228986233, 0.0001762372558005154, 0.00032434286549687386, 0.00020590383792296052, 0.00016373884864151478, 7.893383735790849e-05, 7.528121932409704e-05, 4.0795450331643224e-05, 2.8982227377127856e-05, 2.9523211196647026e-05, 1.5025749235064723e-05, 1.4701029613206629e-05, 2.0762119675055146e-05 ] } } }, "pre_mlp_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "bd772114cfba64f2a56e1ed6a7046489b9ab73852568f07a46a3ac39fd02203c", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.056020382791757584, 0.0742802768945694, 0.1285802572965622, 0.10598119348287582, 0.10133238881826401, 0.10985831171274185, 0.12611564993858337, 0.1125042662024498, 0.07848236709833145, 0.0899009183049202, 0.09790628403425217, 0.09713682532310486, 0.09147685021162033, 0.10080859810113907, 0.09896469116210938, 0.10007023066282272, 0.10974904894828796, 0.10291008651256561, 0.10674569010734558, 0.12148815393447876, 0.055056143552064896, 0.05981758236885071, 0.07101839780807495, 0.08339047431945801, 0.06143829599022865, 0.06018613278865814, 0.06816941499710083, 0.08121132105588913, 0.05916989967226982, 0.07458337396383286, 0.08333029597997665, 0.09104464948177338 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 3.630958900657788e-06, "first_to_last_ratio": 0.20421556767006768, "imbalance_abs_log_ratio": 1.5885791387918327, "last_quartile_mean": 1.778002990704408e-05, "mean": 1.0223539980813712e-05, "normalized": [ 0.8429681862296823, 0.35291594683302396, 0.22484209501334745, 0.36790271245902445, 0.21868611610463093, 0.18545641795638437, 0.23387248761479024, 0.41460977766546225, 0.4054488572719161, 0.3330083345972086, 0.3085788428775866, 0.3192217174198833, 0.3816324391793382, 0.4112131193630629, 0.4344687728023616, 0.46821459950254946, 0.37185344111151275, 0.4758162142971942, 0.41846445085124506, 0.6781708994429974, 2.4552099328163637, 2.8496156807280086, 2.603123697672816, 2.3316927886492507, 2.616234924154862, 2.65437990886044, 2.4006671523655148, 1.672837600494882, 1.585060117848551, 1.2276893902450918, 0.8940161400914827, 0.8621272374795348 ], "peak_layer": 22, "peak_normalized": 2.8496156807280086, "population_cv": 0.8996576825534632, "rest_mean": 7.248915060834541e-06, "spike_contrast": 3.626268804103626, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.6286514548701233e-05, "top_five_layers": [ 22, 26, 25, 23, 21 ], "top_five_spike_overlap": 4 }, "values": [ 8.618118954473175e-06, 3.6080502923141466e-06, 2.298682147738873e-06, 3.7612680898746476e-06, 2.2357462512445636e-06, 1.8960211036755936e-06, 2.3910047275421675e-06, 4.238779638399137e-06, 4.145122602494666e-06, 3.4045240226987517e-06, 3.1547681373922387e-06, 3.263575990786194e-06, 3.9016344999254216e-06, 4.204053766443394e-06, 4.441808869160013e-06, 4.786810677614994e-06, 3.801658522206708e-06, 4.864526090386789e-06, 4.278188043826958e-06, 6.933307304279879e-06, 2.5100936909439042e-05, 2.9133159841876477e-05, 2.661313919816166e-05, 2.383815444773063e-05, 2.674718234629836e-05, 2.7137159122503363e-05, 2.4543316612835042e-05, 1.71023220900679e-05, 1.6204925486817956e-05, 1.2551331565191504e-05, 9.140009751718026e-06, 8.813992280920502e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 1.470679973181177e-05, "first_to_last_ratio": 0.2069929445185519, "imbalance_abs_log_ratio": 1.5750705707490962, "last_quartile_mean": 7.104976339178393e-05, "mean": 4.0840292868438155e-05, "normalized": [ 0.8622316727455452, 0.3538749471728574, 0.22575416255506364, 0.3708511987720451, 0.22255638964420116, 0.1850393452789397, 0.23623438814969275, 0.4242990843945505, 0.41091535564437, 0.3361491695289819, 0.31238992658037873, 0.3212003149521768, 0.3849834068983422, 0.4107834308653127, 0.4382023170687432, 0.4728175618985744, 0.3691518544033243, 0.4681213159064821, 0.4170992962583633, 0.6272493229024287, 2.4438193245013227, 2.8483751664462695, 2.6052355716893176, 2.3350839107599644, 2.618275725913621, 2.653345803911492, 2.3878946201876037, 1.6698710725380324, 1.59563349859018, 1.224673847138525, 0.9002294831629131, 0.8676575135403847 ], "peak_layer": 22, "peak_normalized": 2.8483751664462695, "population_cv": 0.8985289334793173, "rest_mean": 2.8965161365906373e-05, "spike_contrast": 3.6238708169484144, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00010496600298210978, "top_five_layers": [ 22, 26, 25, 23, 21 ], "top_five_spike_overlap": 4 }, "values": [ 3.521379403537139e-05, 1.4452356481342576e-05, 9.219866115017794e-06, 1.5145671568461694e-05, 9.089268132811412e-06, 7.557061053375946e-06, 9.647881597629748e-06, 1.73284988704836e-05, 1.678190346865449e-05, 1.3728430531045888e-05, 1.2758096090692561e-05, 1.3117914932081476e-05, 1.572283508721739e-05, 1.6776515622041188e-05, 1.7896310964715667e-05, 1.9310007701278664e-05, 1.5076269846758805e-05, 1.9118211639579386e-05, 1.703445741441101e-05, 2.5617046048864722e-05, 9.980629693018273e-05, 0.00011632847599685192, 0.00010639858373906463, 9.536551078781486e-05, 0.00010693114745663479, 0.0001083634197129868, 9.752231562742963e-05, 6.819802365498617e-05, 6.516613939311355e-05, 5.0016038585454226e-05, 3.6765635741176084e-05, 3.5435386962490156e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.013042137026786804, "first_to_last_ratio": 0.20699294542918592, "imbalance_abs_log_ratio": 1.5750705663497482, "last_quartile_mean": 0.06300764018669724, "mean": 0.03621758034569211, "normalized": [ 0.8622316598933797, 0.3538749471508598, 0.22575414844580807, 0.37085122221171446, 0.22255637683921708, 0.1850393384194697, 0.236234387422702, 0.42429908068863703, 0.41091534861415707, 0.3361491670478898, 0.3123899060706743, 0.3212003125118064, 0.3849834050274649, 0.41078345814992373, 0.43820230021845763, 0.4728175684224875, 0.36915184353764025, 0.46812130102476074, 0.41709928593336826, 0.6272492932959962, 2.4438193692237777, 2.8483752780000278, 2.605235605762955, 2.3350840258688303, 2.6182758157645756, 2.6533456676353473, 2.3878945209130196, 1.6698711077439734, 1.5956335166208298, 1.2246738042807082, 0.9002294337837535, 0.8676575034757876 ], "peak_layer": 22, "peak_normalized": 2.8483752780000278, "population_cv": 0.8985289457588168, "rest_mean": 0.025686594381652498, "spike_contrast": 3.623871003234085, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.09308490455150605, "top_five_layers": [ 22, 26, 25, 23, 21 ], "top_five_spike_overlap": 4 }, "values": [ 0.031227944418787956, 0.012816494330763817, 0.008176269009709358, 0.013431333936750889, 0.008060453459620476, 0.006701677106320858, 0.008555837906897068, 0.015367086045444012, 0.014882359653711319, 0.012174509465694427, 0.01131400652229786, 0.011633098125457764, 0.01394316740334034, 0.014877582900226116, 0.015870627015829086, 0.01712430827319622, 0.01336978655308485, 0.016954220831394196, 0.015106326900422573, 0.02271745167672634, 0.0885092243552208, 0.10316126048564911, 0.09435532987117767, 0.08457109332084656, 0.09482761472463608, 0.09609775990247726, 0.08648376166820526, 0.06047869101166725, 0.057789985090494156, 0.044354721903800964, 0.03260413184762001, 0.03142445534467697 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 8.568965839117482e-07, "first_to_last_ratio": 0.18051220552416403, "imbalance_abs_log_ratio": 1.7119568828628857, "last_quartile_mean": 4.747028498286454e-06, "mean": 2.6146394631609837e-06, "normalized": [ 0.8589787963589357, 0.3082289930915967, 0.22695141403248437, 0.38502719312209793, 0.20717193772164363, 0.14969905615852777, 0.1917600510841699, 0.2940249836979971, 0.3501028028074662, 0.30509132547078094, 0.2840337285464057, 0.28520738633381276, 0.33814434197742116, 0.362621645282935, 0.4284643797973123, 0.4847838221845296, 0.3528575777552836, 0.44423254880618596, 0.4021648360838168, 0.39838852122464097, 2.445175567674409, 2.9178538331950583, 2.621993890930227, 2.432580720612022, 2.6636416217847234, 2.7243600687846947, 2.580677484419162, 1.8055086156001192, 1.6378764119490026, 1.2592445178301535, 0.939779150965119, 0.9133727747172655 ], "peak_layer": 22, "peak_normalized": 2.9178538331950583, "population_cv": 0.9456027936919144, "rest_mean": 1.8320637688399681e-06, "spike_contrast": 3.7337937296941295, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.840548212494468e-06, "top_five_layers": [ 22, 26, 25, 23, 27 ], "top_five_spike_overlap": 3 }, "values": [ 2.2459198589785956e-06, 8.059076890276629e-07, 5.933961233495211e-07, 1.0067072935271426e-06, 5.416799240265391e-07, 3.9140905983003904e-07, 5.013833970224368e-07, 7.687693255320482e-07, 9.153926043836691e-07, 7.977038194439956e-07, 7.426457955261867e-07, 7.457144874933874e-07, 8.841255407787685e-07, 9.481248639531259e-07, 1.1202798759768484e-06, 1.2675349125856883e-06, 9.225953476743598e-07, 1.1615079529292416e-06, 1.0515160511204158e-06, 1.0416423492642934e-06, 6.39325253359857e-06, 7.629135780007346e-06, 6.855568699393189e-06, 6.360321549436776e-06, 6.964462500036461e-06, 7.123219347704435e-06, 6.747541192453355e-06, 4.7207540774252266e-06, 4.2824563024623785e-06, 3.2924704100878444e-06, 2.4571836547693238e-06, 2.388140501352609e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 9.912464463468496e-06, "first_to_last_ratio": 0.22715332429513255, "imbalance_abs_log_ratio": 1.4821300520824365, "last_quartile_mean": 4.363776974969369e-05, "mean": 2.6559063869058264e-05, "normalized": [ 0.9366358613935047, 0.3792603305202499, 0.18134819814305192, 0.34760749803128715, 0.23747281376634713, 0.19753839084684555, 0.24598063870488243, 0.4599435726461178, 0.4623870697047853, 0.36990754631923695, 0.34903068824582323, 0.37085556359303906, 0.4380455173785828, 0.45529019266842574, 0.48514444968944176, 0.5201735235841253, 0.4152101632813542, 0.5283175891854636, 0.46450504110947355, 0.6293184811026621, 2.502663692008064, 2.9461029153808944, 2.6595225397265994, 2.273367886792679, 2.65101572499286, 2.525147631813147, 2.234627662919172, 1.5898700335733806, 1.426452730852729, 1.112160691183748, 0.8183556047810898, 0.7867397560609359 ], "peak_layer": 22, "peak_normalized": 2.9461029153808944, "population_cv": 0.880175635609515, "rest_mean": 1.8657572430432385e-05, "spike_contrast": 3.7104032636486823, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.9227117637638e-05, "top_five_layers": [ 22, 23, 25, 26, 21 ], "top_five_spike_overlap": 4 }, "values": [ 2.4876171664800495e-05, 1.0072799341287464e-05, 4.816438377019949e-06, 9.2321297415765e-06, 6.307055627985392e-06, 5.246434739092365e-06, 6.533015493914718e-06, 1.2215670722071081e-05, 1.2280567716516089e-05, 9.824398148339242e-06, 9.269928341382183e-06, 9.849576599663123e-06, 1.1634078873612452e-05, 1.2092081306036562e-05, 1.2884982425021008e-05, 1.381532183586387e-05, 1.1027593245671596e-05, 1.4031620594323613e-05, 1.2336819054326043e-05, 1.6714109733584337e-05, 6.646840483881533e-05, 7.824573549441993e-05, 7.063442899379879e-05, 6.037852290319279e-05, 7.040849595796317e-05, 6.70655572321266e-05, 5.934961882303469e-05, 4.222545976517722e-05, 3.788524918491021e-05, 2.953794682980515e-05, 2.1734758774982765e-05, 2.0895071429549716e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 6.6299761840582505e-06, "first_to_last_ratio": 0.25432294789989096, "imbalance_abs_log_ratio": 1.369150371144901, "last_quartile_mean": 2.6069122895933106e-05, "mean": 1.693132426083821e-05, "normalized": [ 1.009812755573757, 0.39968507734198944, 0.16094253648199763, 0.32854895069647194, 0.25477694605768825, 0.2155789505292242, 0.25860705350981517, 0.5046916184590391, 0.515329188356828, 0.40705935830374973, 0.38337515923971865, 0.42269040487394594, 0.4945745095736154, 0.5045501286967757, 0.5247552857381682, 0.5730846754557868, 0.46495437450577576, 0.5946150769636939, 0.5125942030699491, 0.6193454891464502, 2.628840032502442, 3.0610378618268497, 2.7106344368298605, 2.1323334410119363, 2.565140847337925, 2.330500972416668, 2.0157738095858844, 1.503546921244241, 1.3029013314823592, 1.0664512040978233, 0.7882888849242707, 0.7449785141652996 ], "peak_layer": 22, "peak_normalized": 3.0610378618268497, "population_cv": 0.8541047749122238, "rest_mean": 1.1853189545473907e-05, "spike_contrast": 3.7418832756911033, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.435325172380544e-05, "top_five_layers": [ 22, 23, 21, 25, 26 ], "top_five_spike_overlap": 4 }, "values": [ 1.7097467207349837e-05, 6.767197646695422e-06, 2.724970272538485e-06, 5.562768819800112e-06, 4.313711087888805e-06, 3.650037115221494e-06, 4.3785598791146185e-06, 8.54509744385723e-06, 8.725205589144025e-06, 6.892053988849511e-06, 6.4910491346381605e-06, 7.156708306865767e-06, 8.373801392735913e-06, 8.54270183481276e-06, 8.884801900421735e-06, 9.703082469059154e-06, 7.872293281252496e-06, 1.0067620678455569e-05, 8.678898666403256e-06, 1.0486339306226e-05, 4.45097430201713e-05, 5.182742461329326e-05, 4.589463060256094e-05, 3.610322892200202e-05, 4.34312314609997e-05, 3.945846765418537e-05, 3.412972000660375e-05, 2.5457040464971215e-05, 2.2059844923205674e-05, 1.8056431144941598e-05, 1.3346774721867405e-05, 1.2613472790690139e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 2.929049958311225e-05, "first_to_last_ratio": 0.19264517754015853, "imbalance_abs_log_ratio": 1.6469052404570816, "last_quartile_mean": 0.00015204377268673852, "mean": 8.56582414883178e-05, "normalized": [ 0.8435321808032064, 0.34370726972583865, 0.1808664069876229, 0.33993318208269363, 0.2101403998193939, 0.17675988933980996, 0.23091552849720237, 0.4097137534640246, 0.40325567762898656, 0.3332485242329118, 0.315649429844806, 0.31114263500046885, 0.3744856227345994, 0.39208745649289284, 0.44367077395553656, 0.4702768538142674, 0.35548322757045653, 0.45205464826790226, 0.41818583974351736, 0.6083319800365703, 2.3949709614831773, 2.7856906015999057, 2.6171687947428244, 2.3886915605377412, 2.7341447434319206, 2.7222211356577324, 2.584598405216518, 1.8271047030330585, 1.5687480327138839, 1.1190385826225775, 0.8424439925404574, 0.8017372063774949 ], "peak_layer": 22, "peak_normalized": 2.7856906015999057, "population_cv": 0.9245587442879203, "rest_mean": 6.052970898222325e-05, "spike_contrast": 3.6569202255083786, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00022135231702122838, "top_five_layers": [ 22, 25, 26, 23, 27 ], "top_five_spike_overlap": 3 }, "values": [ 7.22554832464084e-05, 2.944136031146627e-05, 1.5492698366870172e-05, 2.911807860073168e-05, 1.8000257114181295e-05, 1.5140941286517773e-05, 1.977981810341589e-05, 3.509535963530652e-05, 3.4542172215878963e-05, 2.8545482564368285e-05, 2.7037975087296218e-05, 2.665193096618168e-05, 3.207777990610339e-05, 3.358552203280851e-05, 3.800405829679221e-05, 4.028308831038885e-05, 3.04500681522768e-05, 3.8722206227248535e-05, 3.582106364774518e-05, 5.2108647651039064e-05, 0.00020514900097623467, 0.000238617358263582, 0.0002241820766357705, 0.00020461111853364855, 0.00023420203069690615, 0.00023318067542277277, 0.00022139215434435755, 0.0001565065758768469, 0.00013437619782052934, 9.58548771450296e-05, 7.21622709534131e-05, 6.867539923405275e-05 ] } } } } }, "parent_learned_round06_exact": true, "step": 4000 }, { "endpoint_exactness": { "checks": { "detached_learned_round06_exact": true, "uniform_all_round06_exact": true }, "passed": true }, "forward_identity_gate": { "comparisons": { "uniform_all": { "activations_exact": true, "logits_exact": true, "loss_exact": true, "mixer_summaries_exact": true }, "uniform_depth_all": { "activations_exact": true, "logits_exact": true, "loss_exact": true, "mixer_summaries_exact": true }, "uniform_except_group_6": { "activations_exact": true, "logits_exact": true, "loss_exact": true, "mixer_summaries_exact": true }, "uniform_except_group_7": { "activations_exact": true, "logits_exact": true, "loss_exact": true, "mixer_summaries_exact": true }, "uniform_except_groups_6_7": { "activations_exact": true, "logits_exact": true, "loss_exact": true, "mixer_summaries_exact": true }, "uniform_group_6_attention_only": { "activations_exact": true, "logits_exact": true, "loss_exact": true, "mixer_summaries_exact": true }, "uniform_group_6_mlp_only": { "activations_exact": true, "logits_exact": true, "loss_exact": true, "mixer_summaries_exact": true }, "uniform_group_6_only": { "activations_exact": true, "logits_exact": true, "loss_exact": true, "mixer_summaries_exact": true }, "uniform_group_7_attention_only": { "activations_exact": true, "logits_exact": true, "loss_exact": true, "mixer_summaries_exact": true }, "uniform_group_7_mlp_only": { "activations_exact": true, "logits_exact": true, "loss_exact": true, "mixer_summaries_exact": true }, "uniform_group_7_only": { "activations_exact": true, "logits_exact": true, "loss_exact": true, "mixer_summaries_exact": true }, "uniform_groups_6_7_only": { "activations_exact": true, "logits_exact": true, "loss_exact": true, "mixer_summaries_exact": true }, "uniform_output_only": { "activations_exact": true, "logits_exact": true, "loss_exact": true, "mixer_summaries_exact": true } }, "passed": true }, "local_matrix": { "detached_learned": { "bits_per_byte": 1.8740129107275643, "forward": { "activation_sha256": { "attention_branch_output": "5e6ecbcd78543fc42d4098a51169dca446d404ce384015643e84d2c78433c949", "mlp_branch_output": "c0b8018c349b1bd09e5d17858eb312a2323da0e92de9651e2bb08ae4adf115c7", "post_attention_state": "90a633f12d984935028fe5d7212be48e7da7b4a0806751939a551938d16dc540", "post_mlp_state": "89f2226456bfe96e245d2aad0f699d32c676f9f53099789cd870253d002402f5", "pre_attention_input": "ee695e78937de01b7128d3866b8fc5dd328ebdf2c159be37918d4f02952d5248", "pre_mlp_input": "e53fa425d89dfeb0e9395eca9bcd73dc0a55314abf45c5b10b43e2cbfa724345" }, "logits_sha256": "bdacba5675f0579cec629bbf01b36c8660ad576cfdf9fef1bea8259eaeac9ec2", "loss_nats": 1.2989667654037476, "mixer_summary_sha256": "eb3ac0f07de9b755d6e67a510ab68d09d9ad666bb7b5d020ba7d8c5055c399b2" }, "loss_nats": 1.2989667654037476, "loss_scale": 1.0, "mixers": [ { "branch": "attention", "entropy_mean": -0.0, "group": 1, "latest_source_mass": 1.0, "layer": 1, "max_source_mass": 1.0, "mixer_index": 0, "normalized_entropy": 1.0, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 1.0, "median": 1.0, "p05": 1.0, "p95": 1.0 } ], "sources": 1 }, { "branch": "mlp", "entropy_mean": 0.22886337339878082, "group": 1, "latest_source_mass": 0.06350618600845337, "layer": 1, "max_source_mass": 0.9364938139915466, "mixer_index": 1, "normalized_entropy": 0.3301800538435402, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.9364938139915466, "median": 0.9452826976776123, "p05": 0.8725647330284119, "p95": 0.9712384939193726 }, { "label": "current_group_partial", "mean": 0.06350618600845337, "median": 0.0547172985970974, "p05": 0.028761589899659157, "p95": 0.12743529677391052 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.603729784488678, "group": 1, "latest_source_mass": 0.4133155345916748, "layer": 2, "max_source_mass": 0.5866844654083252, "mixer_index": 2, "normalized_entropy": 0.8709979661187783, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.5866844654083252, "median": 0.5734395384788513, "p05": 0.29813364148139954, "p95": 0.8843642473220825 }, { "label": "current_group_partial", "mean": 0.4133155345916748, "median": 0.4265604615211487, "p05": 0.11563576757907867, "p95": 0.7018663287162781 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.3967982828617096, "group": 1, "latest_source_mass": 0.14110636711120605, "layer": 2, "max_source_mass": 0.858893632888794, "mixer_index": 3, "normalized_entropy": 0.5724589149178446, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.858893632888794, "median": 0.8705973029136658, "p05": 0.7599387168884277, "p95": 0.9270997643470764 }, { "label": "current_group_partial", "mean": 0.14110636711120605, "median": 0.12940272688865662, "p05": 0.07290027290582657, "p95": 0.2400612235069275 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6852496862411499, "group": 1, "latest_source_mass": 0.518975019454956, "layer": 3, "max_source_mass": 0.518975019454956, "mixer_index": 4, "normalized_entropy": 0.9886063241108252, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.48102498054504395, "median": 0.48102718591690063, "p05": 0.38786420226097107, "p95": 0.580143928527832 }, { "label": "current_group_partial", "mean": 0.518975019454956, "median": 0.5189728140830994, "p05": 0.41985607147216797, "p95": 0.6121357679367065 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.5890734195709229, "group": 1, "latest_source_mass": 0.3123423159122467, "layer": 3, "max_source_mass": 0.6876576542854309, "mixer_index": 5, "normalized_entropy": 0.8498533011344741, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.6876576542854309, "median": 0.6909762620925903, "p05": 0.48803940415382385, "p95": 0.8900076746940613 }, { "label": "current_group_partial", "mean": 0.3123423159122467, "median": 0.3090237081050873, "p05": 0.10999235510826111, "p95": 0.5119606256484985 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6873323321342468, "group": 1, "latest_source_mass": 0.4831485152244568, "layer": 4, "max_source_mass": 0.5168514251708984, "mixer_index": 6, "normalized_entropy": 0.9916109470127238, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.5168514251708984, "median": 0.5219584703445435, "p05": 0.4251725673675537, "p95": 0.5947866439819336 }, { "label": "current_group_partial", "mean": 0.4831485152244568, "median": 0.47804147005081177, "p05": 0.405213326215744, "p95": 0.5748274326324463 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.487301766872406, "group": 1, "latest_source_mass": 0.2060384750366211, "layer": 4, "max_source_mass": 0.7939615249633789, "mixer_index": 7, "normalized_entropy": 0.7030278424832499, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.7939615249633789, "median": 0.7953404188156128, "p05": 0.6590759754180908, "p95": 0.9329437017440796 }, { "label": "current_group_partial", "mean": 0.2060384750366211, "median": 0.20465964078903198, "p05": 0.06705628335475922, "p95": 0.3409239649772644 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.674697756767273, "group": 2, "latest_source_mass": 0.42952626943588257, "layer": 5, "max_source_mass": 0.5704737305641174, "mixer_index": 8, "normalized_entropy": 0.9733831077870527, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.5704737305641174, "median": 0.5696132779121399, "p05": 0.4695202708244324, "p95": 0.6750621199607849 }, { "label": "completed_group_1", "mean": 0.42952626943588257, "median": 0.4303867816925049, "p05": 0.3249378800392151, "p95": 0.5304797887802124 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.9691112041473389, "group": 2, "latest_source_mass": 0.2681158185005188, "layer": 5, "max_source_mass": 0.5506307482719421, "mixer_index": 9, "normalized_entropy": 0.8821230329784768, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.5506307482719421, "median": 0.543993353843689, "p05": 0.41262108087539673, "p95": 0.7161091566085815 }, { "label": "completed_group_1", "mean": 0.18125343322753906, "median": 0.1794438362121582, "p05": 0.11575750261545181, "p95": 0.25031501054763794 }, { "label": "current_group_partial", "mean": 0.2681158185005188, "median": 0.2664111256599426, "p05": 0.1564716249704361, "p95": 0.3817881643772125 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0745112895965576, "group": 2, "latest_source_mass": 0.412794291973114, "layer": 6, "max_source_mass": 0.412794291973114, "mixer_index": 10, "normalized_entropy": 0.9780623252441762, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.34262996912002563, "median": 0.3412924110889435, "p05": 0.29556700587272644, "p95": 0.39125555753707886 }, { "label": "completed_group_1", "mean": 0.24457570910453796, "median": 0.24382859468460083, "p05": 0.22120657563209534, "p95": 0.2734297513961792 }, { "label": "current_group_partial", "mean": 0.412794291973114, "median": 0.4151499271392822, "p05": 0.37735897302627563, "p95": 0.43749260902404785 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0074045658111572, "group": 2, "latest_source_mass": 0.3775783181190491, "layer": 6, "max_source_mass": 0.4399106502532959, "mixer_index": 11, "normalized_entropy": 0.9169791528842925, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.4399106502532959, "median": 0.44856905937194824, "p05": 0.22063353657722473, "p95": 0.6359403133392334 }, { "label": "completed_group_1", "mean": 0.18251106142997742, "median": 0.18562838435173035, "p05": 0.1196163073182106, "p95": 0.23045453429222107 }, { "label": "current_group_partial", "mean": 0.3775783181190491, "median": 0.3675365447998047, "p05": 0.23817461729049683, "p95": 0.5557451844215393 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.060497522354126, "group": 2, "latest_source_mass": 0.4025024175643921, "layer": 7, "max_source_mass": 0.4025024175643921, "mixer_index": 12, "normalized_entropy": 0.9653064445872968, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.3786153197288513, "median": 0.3907395303249359, "p05": 0.28039106726646423, "p95": 0.4436405599117279 }, { "label": "completed_group_1", "mean": 0.2188822478055954, "median": 0.21226292848587036, "p05": 0.1885872334241867, "p95": 0.2672030031681061 }, { "label": "current_group_partial", "mean": 0.4025024175643921, "median": 0.39815032482147217, "p05": 0.3533596992492676, "p95": 0.460100919008255 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 0.9941019415855408, "group": 2, "latest_source_mass": 0.4801652431488037, "layer": 7, "max_source_mass": 0.4801652431488037, "mixer_index": 13, "normalized_entropy": 0.9048705824970601, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.32296955585479736, "median": 0.2803739905357361, "p05": 0.1784922033548355, "p95": 0.6440245509147644 }, { "label": "completed_group_1", "mean": 0.19686520099639893, "median": 0.2053825855255127, "p05": 0.09813719242811203, "p95": 0.2504696547985077 }, { "label": "current_group_partial", "mean": 0.4801652431488037, "median": 0.5057140588760376, "p05": 0.2529003322124481, "p95": 0.598870575428009 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0503522157669067, "group": 2, "latest_source_mass": 0.4001518487930298, "layer": 8, "max_source_mass": 0.40130966901779175, "mixer_index": 14, "normalized_entropy": 0.9560717885654542, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.40130966901779175, "median": 0.396759569644928, "p05": 0.3479611873626709, "p95": 0.4733521044254303 }, { "label": "completed_group_1", "mean": 0.19853851199150085, "median": 0.19769755005836487, "p05": 0.17136667668819427, "p95": 0.22788912057876587 }, { "label": "current_group_partial", "mean": 0.4001518487930298, "median": 0.40291696786880493, "p05": 0.3506356179714203, "p95": 0.43941178917884827 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0129402875900269, "group": 2, "latest_source_mass": 0.42161279916763306, "layer": 8, "max_source_mass": 0.42161279916763306, "mixer_index": 15, "normalized_entropy": 0.9220179839951123, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.4093267023563385, "median": 0.4046899676322937, "p05": 0.2895161807537079, "p95": 0.5762131810188293 }, { "label": "completed_group_1", "mean": 0.16906049847602844, "median": 0.16888955235481262, "p05": 0.11233551055192947, "p95": 0.22477403283119202 }, { "label": "current_group_partial", "mean": 0.42161279916763306, "median": 0.4227708578109741, "p05": 0.3131049573421478, "p95": 0.519075334072113 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0774019956588745, "group": 3, "latest_source_mass": 0.3560434877872467, "layer": 9, "max_source_mass": 0.38510820269584656, "mixer_index": 16, "normalized_entropy": 0.9806935592947451, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.38510820269584656, "median": 0.3892287313938141, "p05": 0.2934730052947998, "p95": 0.4718902111053467 }, { "label": "completed_group_1", "mean": 0.25884830951690674, "median": 0.25734007358551025, "p05": 0.22217319905757904, "p95": 0.29551801085472107 }, { "label": "completed_group_2", "mean": 0.3560434877872467, "median": 0.35412120819091797, "p05": 0.29444968700408936, "p95": 0.41908568143844604 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.2740377187728882, "group": 3, "latest_source_mass": 0.3633861243724823, "layer": 9, "max_source_mass": 0.3633861243724823, "mixer_index": 17, "normalized_entropy": 0.9190239493895668, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.29550763964653015, "median": 0.30360478162765503, "p05": 0.16904757916927338, "p95": 0.45146965980529785 }, { "label": "completed_group_1", "mean": 0.09541244804859161, "median": 0.09486871212720871, "p05": 0.061360105872154236, "p95": 0.13037073612213135 }, { "label": "completed_group_2", "mean": 0.24569380283355713, "median": 0.24333015084266663, "p05": 0.1698218435049057, "p95": 0.31589236855506897 }, { "label": "current_group_partial", "mean": 0.3633861243724823, "median": 0.36472851037979126, "p05": 0.2795379161834717, "p95": 0.4399876892566681 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.360302448272705, "group": 3, "latest_source_mass": 0.28569501638412476, "layer": 10, "max_source_mass": 0.3109527826309204, "mixer_index": 18, "normalized_entropy": 0.9812507981160736, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.3109527826309204, "median": 0.32172977924346924, "p05": 0.23520725965499878, "p95": 0.37052470445632935 }, { "label": "completed_group_1", "mean": 0.20984819531440735, "median": 0.20958667993545532, "p05": 0.1891367882490158, "p95": 0.23184512555599213 }, { "label": "completed_group_2", "mean": 0.19350400567054749, "median": 0.19153150916099548, "p05": 0.16607457399368286, "p95": 0.22877082228660583 }, { "label": "current_group_partial", "mean": 0.28569501638412476, "median": 0.28060320019721985, "p05": 0.23824527859687805, "p95": 0.3439165949821472 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.2722704410552979, "group": 3, "latest_source_mass": 0.29912811517715454, "layer": 10, "max_source_mass": 0.35036465525627136, "mixer_index": 19, "normalized_entropy": 0.9177491279900463, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.35036465525627136, "median": 0.36210548877716064, "p05": 0.12807878851890564, "p95": 0.5797606706619263 }, { "label": "completed_group_1", "mean": 0.12631990015506744, "median": 0.1264108121395111, "p05": 0.06977735459804535, "p95": 0.1755298376083374 }, { "label": "completed_group_2", "mean": 0.22418735921382904, "median": 0.22563719749450684, "p05": 0.1176326721906662, "p95": 0.32001155614852905 }, { "label": "current_group_partial", "mean": 0.29912811517715454, "median": 0.28589946031570435, "p05": 0.193109393119812, "p95": 0.44119665026664734 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3558036088943481, "group": 3, "latest_source_mass": 0.314197838306427, "layer": 11, "max_source_mass": 0.314197838306427, "mixer_index": 20, "normalized_entropy": 0.978005571485618, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.298976868391037, "median": 0.2961939573287964, "p05": 0.24950163066387177, "p95": 0.35835936665534973 }, { "label": "completed_group_1", "mean": 0.1941717267036438, "median": 0.1918032318353653, "p05": 0.16809208691120148, "p95": 0.22895419597625732 }, { "label": "completed_group_2", "mean": 0.1926535815000534, "median": 0.19306811690330505, "p05": 0.16983944177627563, "p95": 0.2155935913324356 }, { "label": "current_group_partial", "mean": 0.314197838306427, "median": 0.31541913747787476, "p05": 0.2673022747039795, "p95": 0.35941022634506226 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.2696614265441895, "group": 3, "latest_source_mass": 0.2862822115421295, "layer": 11, "max_source_mass": 0.3667730689048767, "mixer_index": 21, "normalized_entropy": 0.9158671218416545, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.3667730689048767, "median": 0.4067659378051758, "p05": 0.10593366622924805, "p95": 0.5672274827957153 }, { "label": "completed_group_1", "mean": 0.15310388803482056, "median": 0.15050218999385834, "p05": 0.08651046454906464, "p95": 0.22459912300109863 }, { "label": "completed_group_2", "mean": 0.19384080171585083, "median": 0.18732509016990662, "p05": 0.1267009675502777, "p95": 0.281107634305954 }, { "label": "current_group_partial", "mean": 0.2862822115421295, "median": 0.26271265745162964, "p05": 0.17024393379688263, "p95": 0.475044310092926 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3606936931610107, "group": 3, "latest_source_mass": 0.3028184771537781, "layer": 12, "max_source_mass": 0.3028184771537781, "mixer_index": 22, "normalized_entropy": 0.9815330216461395, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.298227459192276, "median": 0.2954302132129669, "p05": 0.25287362933158875, "p95": 0.35405808687210083 }, { "label": "completed_group_1", "mean": 0.18611131608486176, "median": 0.1850002110004425, "p05": 0.15901757776737213, "p95": 0.21725836396217346 }, { "label": "completed_group_2", "mean": 0.21284276247024536, "median": 0.21402311325073242, "p05": 0.18819917738437653, "p95": 0.23418807983398438 }, { "label": "current_group_partial", "mean": 0.3028184771537781, "median": 0.3041122555732727, "p05": 0.25661808252334595, "p95": 0.34458184242248535 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.269316554069519, "group": 3, "latest_source_mass": 0.2653314769268036, "layer": 12, "max_source_mass": 0.4081852436065674, "mixer_index": 23, "normalized_entropy": 0.9156183489371815, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.4081852436065674, "median": 0.4254342317581177, "p05": 0.20765496790409088, "p95": 0.587238073348999 }, { "label": "completed_group_1", "mean": 0.14420127868652344, "median": 0.1438048928976059, "p05": 0.1064920499920845, "p95": 0.1871720254421234 }, { "label": "completed_group_2", "mean": 0.18228203058242798, "median": 0.17507661879062653, "p05": 0.12183455377817154, "p95": 0.26505547761917114 }, { "label": "current_group_partial", "mean": 0.2653314769268036, "median": 0.24924281239509583, "p05": 0.15609969198703766, "p95": 0.4061311185359955 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3636136054992676, "group": 4, "latest_source_mass": 0.3136237859725952, "layer": 13, "max_source_mass": 0.3136237859725952, "mixer_index": 24, "normalized_entropy": 0.9836392931712563, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.27427661418914795, "median": 0.27403637766838074, "p05": 0.22928935289382935, "p95": 0.319773405790329 }, { "label": "completed_group_1", "mean": 0.21341991424560547, "median": 0.20697325468063354, "p05": 0.18552586436271667, "p95": 0.26688331365585327 }, { "label": "completed_group_2", "mean": 0.19867971539497375, "median": 0.197545126080513, "p05": 0.1794913411140442, "p95": 0.222100168466568 }, { "label": "completed_group_3", "mean": 0.3136237859725952, "median": 0.31875866651535034, "p05": 0.24156072735786438, "p95": 0.3695686459541321 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.570773959159851, "group": 4, "latest_source_mass": 0.2294243425130844, "layer": 13, "max_source_mass": 0.2658740282058716, "mixer_index": 25, "normalized_entropy": 0.9759767351225285, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.2658740282058716, "median": 0.2747163474559784, "p05": 0.17927880585193634, "p95": 0.3512483835220337 }, { "label": "completed_group_1", "mean": 0.13419361412525177, "median": 0.13374894857406616, "p05": 0.11072897166013718, "p95": 0.1586270034313202 }, { "label": "completed_group_2", "mean": 0.1907704919576645, "median": 0.19074805080890656, "p05": 0.14699944853782654, "p95": 0.23316305875778198 }, { "label": "completed_group_3", "mean": 0.17973749339580536, "median": 0.17616114020347595, "p05": 0.1363743096590042, "p95": 0.23202598094940186 }, { "label": "current_group_partial", "mean": 0.2294243425130844, "median": 0.22995705902576447, "p05": 0.17975632846355438, "p95": 0.2789157032966614 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.5560038089752197, "group": 4, "latest_source_mass": 0.32468360662460327, "layer": 14, "max_source_mass": 0.32468360662460327, "mixer_index": 26, "normalized_entropy": 0.9667995248241249, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.20477107167243958, "median": 0.206403911113739, "p05": 0.16619175672531128, "p95": 0.2431035190820694 }, { "label": "completed_group_1", "mean": 0.15721669793128967, "median": 0.15658551454544067, "p05": 0.13878680765628815, "p95": 0.17778633534908295 }, { "label": "completed_group_2", "mean": 0.13811248540878296, "median": 0.13743364810943604, "p05": 0.12352626770734787, "p95": 0.15553249418735504 }, { "label": "completed_group_3", "mean": 0.1752161681652069, "median": 0.17309701442718506, "p05": 0.15223677456378937, "p95": 0.20474471151828766 }, { "label": "current_group_partial", "mean": 0.32468360662460327, "median": 0.32497870922088623, "p05": 0.2946181893348694, "p95": 0.35203880071640015 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.5659973621368408, "group": 4, "latest_source_mass": 0.26703086495399475, "layer": 14, "max_source_mass": 0.26703086495399475, "mixer_index": 27, "normalized_entropy": 0.9730088685238187, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.20531430840492249, "median": 0.20229148864746094, "p05": 0.10698921978473663, "p95": 0.3393132984638214 }, { "label": "completed_group_1", "mean": 0.15986156463623047, "median": 0.15952220559120178, "p05": 0.11969029903411865, "p95": 0.20180393755435944 }, { "label": "completed_group_2", "mean": 0.21557100117206573, "median": 0.21654027700424194, "p05": 0.1673085242509842, "p95": 0.2610440254211426 }, { "label": "completed_group_3", "mean": 0.15222224593162537, "median": 0.1510905921459198, "p05": 0.11800260841846466, "p95": 0.18893754482269287 }, { "label": "current_group_partial", "mean": 0.26703086495399475, "median": 0.258539617061615, "p05": 0.19330690801143646, "p95": 0.36521556973457336 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.5552496910095215, "group": 4, "latest_source_mass": 0.31774693727493286, "layer": 15, "max_source_mass": 0.31774693727493286, "mixer_index": 28, "normalized_entropy": 0.9663309649872576, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.2162160575389862, "median": 0.2145705670118332, "p05": 0.17727568745613098, "p95": 0.26878347992897034 }, { "label": "completed_group_1", "mean": 0.17982497811317444, "median": 0.18024154007434845, "p05": 0.15453973412513733, "p95": 0.2030555009841919 }, { "label": "completed_group_2", "mean": 0.1307673156261444, "median": 0.13107559084892273, "p05": 0.11463287472724915, "p95": 0.14630235731601715 }, { "label": "completed_group_3", "mean": 0.1554446816444397, "median": 0.15516987442970276, "p05": 0.13627474009990692, "p95": 0.1754152774810791 }, { "label": "current_group_partial", "mean": 0.31774693727493286, "median": 0.319039523601532, "p05": 0.27606821060180664, "p95": 0.35650834441185 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.5068919658660889, "group": 4, "latest_source_mass": 0.3298327624797821, "layer": 15, "max_source_mass": 0.3298327624797821, "mixer_index": 29, "normalized_entropy": 0.9362846209998111, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.20646697282791138, "median": 0.18775638937950134, "p05": 0.0806753933429718, "p95": 0.4274584949016571 }, { "label": "completed_group_1", "mean": 0.1197461187839508, "median": 0.11989086866378784, "p05": 0.0809398666024208, "p95": 0.15743419528007507 }, { "label": "completed_group_2", "mean": 0.19215428829193115, "median": 0.19620640575885773, "p05": 0.1148109883069992, "p95": 0.2491212636232376 }, { "label": "completed_group_3", "mean": 0.15179988741874695, "median": 0.15212899446487427, "p05": 0.10697861760854721, "p95": 0.19542087614536285 }, { "label": "current_group_partial", "mean": 0.3298327624797821, "median": 0.32837340235710144, "p05": 0.22746729850769043, "p95": 0.4339301884174347 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.5435223579406738, "group": 4, "latest_source_mass": 0.3224271535873413, "layer": 16, "max_source_mass": 0.3224271535873413, "mixer_index": 30, "normalized_entropy": 0.9590443632623663, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.22410975396633148, "median": 0.22760415077209473, "p05": 0.14593973755836487, "p95": 0.30093055963516235 }, { "label": "completed_group_1", "mean": 0.16986176371574402, "median": 0.16963240504264832, "p05": 0.13569067418575287, "p95": 0.20651353895664215 }, { "label": "completed_group_2", "mean": 0.13080403208732605, "median": 0.13075244426727295, "p05": 0.10945598781108856, "p95": 0.15187400579452515 }, { "label": "completed_group_3", "mean": 0.15279728174209595, "median": 0.15036612749099731, "p05": 0.12810495495796204, "p95": 0.18468116223812103 }, { "label": "current_group_partial", "mean": 0.3224271535873413, "median": 0.3164973258972168, "p05": 0.2600978910923004, "p95": 0.40530598163604736 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.4949920177459717, "group": 4, "latest_source_mass": 0.31847020983695984, "layer": 16, "max_source_mass": 0.31847020983695984, "mixer_index": 31, "normalized_entropy": 0.9288907675133354, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.27174079418182373, "median": 0.2585451304912567, "p05": 0.13639721274375916, "p95": 0.4698348045349121 }, { "label": "completed_group_1", "mean": 0.12178683280944824, "median": 0.11714950203895569, "p05": 0.08594125509262085, "p95": 0.1723237782716751 }, { "label": "completed_group_2", "mean": 0.1529826819896698, "median": 0.15223106741905212, "p05": 0.10371473431587219, "p95": 0.203575000166893 }, { "label": "completed_group_3", "mean": 0.1350194811820984, "median": 0.13363957405090332, "p05": 0.08964549005031586, "p95": 0.18468207120895386 }, { "label": "current_group_partial", "mean": 0.31847020983695984, "median": 0.3185216188430786, "p05": 0.21444711089134216, "p95": 0.4301546812057495 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.544264793395996, "group": 5, "latest_source_mass": 0.2861037850379944, "layer": 17, "max_source_mass": 0.2861037850379944, "mixer_index": 32, "normalized_entropy": 0.9595056643474137, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.24016115069389343, "median": 0.23868846893310547, "p05": 0.18435582518577576, "p95": 0.2983453571796417 }, { "label": "completed_group_1", "mean": 0.20730282366275787, "median": 0.1998077780008316, "p05": 0.16255050897598267, "p95": 0.28107208013534546 }, { "label": "completed_group_2", "mean": 0.09692573547363281, "median": 0.0967404842376709, "p05": 0.08098706603050232, "p95": 0.11221462488174438 }, { "label": "completed_group_3", "mean": 0.1695065200328827, "median": 0.1696585714817047, "p05": 0.136396586894989, "p95": 0.20130634307861328 }, { "label": "completed_group_4", "mean": 0.2861037850379944, "median": 0.28883200883865356, "p05": 0.22401469945907593, "p95": 0.3409370183944702 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.686682939529419, "group": 5, "latest_source_mass": 0.12281867861747742, "layer": 17, "max_source_mass": 0.2885604202747345, "mixer_index": 33, "normalized_entropy": 0.9413556721740635, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.16755425930023193, "median": 0.1327776312828064, "p05": 0.0790192037820816, "p95": 0.3667412996292114 }, { "label": "completed_group_1", "mean": 0.10962115228176117, "median": 0.10525335371494293, "p05": 0.07840172946453094, "p95": 0.15405339002609253 }, { "label": "completed_group_2", "mean": 0.1783454418182373, "median": 0.17896884679794312, "p05": 0.10802385956048965, "p95": 0.24048911035060883 }, { "label": "completed_group_3", "mean": 0.13310004770755768, "median": 0.13033035397529602, "p05": 0.0904831737279892, "p95": 0.17934636771678925 }, { "label": "completed_group_4", "mean": 0.2885604202747345, "median": 0.2831825315952301, "p05": 0.16967125236988068, "p95": 0.4189499616622925 }, { "label": "current_group_partial", "mean": 0.12281867861747742, "median": 0.12062528729438782, "p05": 0.07461707293987274, "p95": 0.1782219111919403 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.7070834636688232, "group": 5, "latest_source_mass": 0.2988377809524536, "layer": 18, "max_source_mass": 0.2988377809524536, "mixer_index": 34, "normalized_entropy": 0.9527414214834803, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.181177020072937, "median": 0.18419909477233887, "p05": 0.13624237477779388, "p95": 0.22192169725894928 }, { "label": "completed_group_1", "mean": 0.14557164907455444, "median": 0.144476979970932, "p05": 0.1293315887451172, "p95": 0.16486899554729462 }, { "label": "completed_group_2", "mean": 0.08318156749010086, "median": 0.0820261612534523, "p05": 0.07088497281074524, "p95": 0.0990293100476265 }, { "label": "completed_group_3", "mean": 0.11424209177494049, "median": 0.11267061531543732, "p05": 0.0979941189289093, "p95": 0.1353992223739624 }, { "label": "completed_group_4", "mean": 0.17698991298675537, "median": 0.17597860097885132, "p05": 0.14837387204170227, "p95": 0.21050144731998444 }, { "label": "current_group_partial", "mean": 0.2988377809524536, "median": 0.2989804744720459, "p05": 0.2721370458602905, "p95": 0.32572072744369507 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.7419235706329346, "group": 5, "latest_source_mass": 0.26016175746917725, "layer": 18, "max_source_mass": 0.26016175746917725, "mixer_index": 35, "normalized_entropy": 0.972186055410333, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1532188355922699, "median": 0.1451343297958374, "p05": 0.11514341086149216, "p95": 0.22103679180145264 }, { "label": "completed_group_1", "mean": 0.11420352011919022, "median": 0.11261284351348877, "p05": 0.0928458571434021, "p95": 0.14123988151550293 }, { "label": "completed_group_2", "mean": 0.16597424447536469, "median": 0.1655328869819641, "p05": 0.13871800899505615, "p95": 0.19611573219299316 }, { "label": "completed_group_3", "mean": 0.1250365972518921, "median": 0.12369821965694427, "p05": 0.10375828295946121, "p95": 0.14912742376327515 }, { "label": "completed_group_4", "mean": 0.18140508234500885, "median": 0.1796584278345108, "p05": 0.14511024951934814, "p95": 0.22286373376846313 }, { "label": "current_group_partial", "mean": 0.26016175746917725, "median": 0.26266711950302124, "p05": 0.1889389455318451, "p95": 0.3234381675720215 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.7127439975738525, "group": 5, "latest_source_mass": 0.2907913327217102, "layer": 19, "max_source_mass": 0.2907913327217102, "mixer_index": 36, "normalized_entropy": 0.9559006256078307, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.18731768429279327, "median": 0.19042012095451355, "p05": 0.13879702985286713, "p95": 0.23792000114917755 }, { "label": "completed_group_1", "mean": 0.16715484857559204, "median": 0.1667211651802063, "p05": 0.14574480056762695, "p95": 0.19026139378547668 }, { "label": "completed_group_2", "mean": 0.08496320247650146, "median": 0.08452782034873962, "p05": 0.07410907745361328, "p95": 0.097386933863163 }, { "label": "completed_group_3", "mean": 0.11362075805664062, "median": 0.11248823255300522, "p05": 0.0977015346288681, "p95": 0.13219426572322845 }, { "label": "completed_group_4", "mean": 0.15615218877792358, "median": 0.15540224313735962, "p05": 0.1289147436618805, "p95": 0.18598109483718872 }, { "label": "current_group_partial", "mean": 0.2907913327217102, "median": 0.2900415062904358, "p05": 0.25822532176971436, "p95": 0.32475030422210693 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.6076834201812744, "group": 5, "latest_source_mass": 0.3738824725151062, "layer": 19, "max_source_mass": 0.3738824725151062, "mixer_index": 37, "normalized_entropy": 0.8972652009334232, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.15032559633255005, "median": 0.12544110417366028, "p05": 0.06976518034934998, "p95": 0.35840943455696106 }, { "label": "completed_group_1", "mean": 0.08624927699565887, "median": 0.08602266013622284, "p05": 0.061998721212148666, "p95": 0.11138717085123062 }, { "label": "completed_group_2", "mean": 0.15736976265907288, "median": 0.15802210569381714, "p05": 0.10121746361255646, "p95": 0.20687000453472137 }, { "label": "completed_group_3", "mean": 0.08584291487932205, "median": 0.0827617198228836, "p05": 0.060561805963516235, "p95": 0.11351694166660309 }, { "label": "completed_group_4", "mean": 0.14632999897003174, "median": 0.1426895558834076, "p05": 0.10106660425662994, "p95": 0.20179018378257751 }, { "label": "current_group_partial", "mean": 0.3738824725151062, "median": 0.3799295425415039, "p05": 0.23631010949611664, "p95": 0.4884527325630188 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.709290862083435, "group": 5, "latest_source_mass": 0.2901708781719208, "layer": 20, "max_source_mass": 0.2901708781719208, "mixer_index": 38, "normalized_entropy": 0.9539733939957076, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.18991519510746002, "median": 0.1924753338098526, "p05": 0.14201581478118896, "p95": 0.23709386587142944 }, { "label": "completed_group_1", "mean": 0.17091524600982666, "median": 0.17006167769432068, "p05": 0.15297070145606995, "p95": 0.19153237342834473 }, { "label": "completed_group_2", "mean": 0.07822999358177185, "median": 0.07778562605381012, "p05": 0.06707640737295151, "p95": 0.09089364111423492 }, { "label": "completed_group_3", "mean": 0.116765096783638, "median": 0.11516137421131134, "p05": 0.10162825882434845, "p95": 0.13653048872947693 }, { "label": "completed_group_4", "mean": 0.1540035903453827, "median": 0.15287795662879944, "p05": 0.12593276798725128, "p95": 0.18590238690376282 }, { "label": "current_group_partial", "mean": 0.2901708781719208, "median": 0.29104921221733093, "p05": 0.25643545389175415, "p95": 0.3205650746822357 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.6690887212753296, "group": 5, "latest_source_mass": 0.3342471718788147, "layer": 20, "max_source_mass": 0.3342471718788147, "mixer_index": 39, "normalized_entropy": 0.9315361520005944, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.19029226899147034, "median": 0.19644704461097717, "p05": 0.09456793963909149, "p95": 0.30468082427978516 }, { "label": "completed_group_1", "mean": 0.1100241169333458, "median": 0.10808776319026947, "p05": 0.09035106748342514, "p95": 0.1359245330095291 }, { "label": "completed_group_2", "mean": 0.11471512168645859, "median": 0.11494918912649155, "p05": 0.08799737691879272, "p95": 0.14024731516838074 }, { "label": "completed_group_3", "mean": 0.09999909996986389, "median": 0.09682698547840118, "p05": 0.07921088486909866, "p95": 0.1264391839504242 }, { "label": "completed_group_4", "mean": 0.1507222205400467, "median": 0.1476232260465622, "p05": 0.11397356539964676, "p95": 0.19991227984428406 }, { "label": "current_group_partial", "mean": 0.3342471718788147, "median": 0.33802270889282227, "p05": 0.2548266053199768, "p95": 0.40130624175071716 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.7458693981170654, "group": 6, "latest_source_mass": 0.2361394464969635, "layer": 21, "max_source_mass": 0.2361394464969635, "mixer_index": 40, "normalized_entropy": 0.9743882636597644, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.20528709888458252, "median": 0.2140858769416809, "p05": 0.13168928027153015, "p95": 0.27098023891448975 }, { "label": "completed_group_1", "mean": 0.1722964197397232, "median": 0.17238952219486237, "p05": 0.15053534507751465, "p95": 0.19365043938159943 }, { "label": "completed_group_2", "mean": 0.10435444861650467, "median": 0.10361101478338242, "p05": 0.08888213336467743, "p95": 0.12212536484003067 }, { "label": "completed_group_3", "mean": 0.12075909972190857, "median": 0.1180175170302391, "p05": 0.10358668118715286, "p95": 0.14458929002285004 }, { "label": "completed_group_4", "mean": 0.16116347908973694, "median": 0.15841388702392578, "p05": 0.1337081342935562, "p95": 0.19791515171527863 }, { "label": "completed_group_5", "mean": 0.2361394464969635, "median": 0.23709020018577576, "p05": 0.20446214079856873, "p95": 0.26509594917297363 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.4612125158309937, "group": 6, "latest_source_mass": 0.5543455481529236, "layer": 21, "max_source_mass": 0.5543455481529236, "mixer_index": 41, "normalized_entropy": 0.7509146897354808, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.08934466540813446, "median": 0.07911523431539536, "p05": 0.04488275945186615, "p95": 0.1737232506275177 }, { "label": "completed_group_1", "mean": 0.054379627108573914, "median": 0.05344422906637192, "p05": 0.04221750423312187, "p95": 0.06924191862344742 }, { "label": "completed_group_2", "mean": 0.07567165046930313, "median": 0.07438549399375916, "p05": 0.05811355262994766, "p95": 0.09632256627082825 }, { "label": "completed_group_3", "mean": 0.05758609622716904, "median": 0.05612001568078995, "p05": 0.044792283326387405, "p95": 0.07347827404737473 }, { "label": "completed_group_4", "mean": 0.07884950935840607, "median": 0.07757881283760071, "p05": 0.05758974328637123, "p95": 0.10452556610107422 }, { "label": "completed_group_5", "mean": 0.08982288837432861, "median": 0.08912864327430725, "p05": 0.0663854256272316, "p95": 0.11604885756969452 }, { "label": "current_group_partial", "mean": 0.5543455481529236, "median": 0.5596089363098145, "p05": 0.45455610752105713, "p95": 0.6367484331130981 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.7241814136505127, "group": 6, "latest_source_mass": 0.39484071731567383, "layer": 22, "max_source_mass": 0.39484071731567383, "mixer_index": 42, "normalized_entropy": 0.886053970419732, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.13360010087490082, "median": 0.14072942733764648, "p05": 0.07699814438819885, "p95": 0.17506597936153412 }, { "label": "completed_group_1", "mean": 0.10609637200832367, "median": 0.10586297512054443, "p05": 0.08648421615362167, "p95": 0.12602579593658447 }, { "label": "completed_group_2", "mean": 0.06363911926746368, "median": 0.06324908882379532, "p05": 0.054980404675006866, "p95": 0.07316673547029495 }, { "label": "completed_group_3", "mean": 0.07426095008850098, "median": 0.07362888753414154, "p05": 0.06318790465593338, "p95": 0.08665015548467636 }, { "label": "completed_group_4", "mean": 0.09383076429367065, "median": 0.09294568747282028, "p05": 0.07638302445411682, "p95": 0.11445100605487823 }, { "label": "completed_group_5", "mean": 0.13373197615146637, "median": 0.13327741622924805, "p05": 0.11399447172880173, "p95": 0.15641780197620392 }, { "label": "current_group_partial", "mean": 0.39484071731567383, "median": 0.3879151940345764, "p05": 0.3302172124385834, "p95": 0.4852263331413269 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.3979300260543823, "group": 6, "latest_source_mass": 0.5830056071281433, "layer": 22, "max_source_mass": 0.5830056071281433, "mixer_index": 43, "normalized_entropy": 0.7183939231382495, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.08959861099720001, "median": 0.07544538378715515, "p05": 0.04165133833885193, "p95": 0.20099139213562012 }, { "label": "completed_group_1", "mean": 0.05414731800556183, "median": 0.05336771905422211, "p05": 0.04069708660244942, "p95": 0.07046990841627121 }, { "label": "completed_group_2", "mean": 0.07668881118297577, "median": 0.07607124000787735, "p05": 0.0555989071726799, "p95": 0.09980960935354233 }, { "label": "completed_group_3", "mean": 0.05603286623954773, "median": 0.054573021829128265, "p05": 0.04162490367889404, "p95": 0.0730118378996849 }, { "label": "completed_group_4", "mean": 0.06979228556156158, "median": 0.06884932518005371, "p05": 0.04576495662331581, "p95": 0.09806846082210541 }, { "label": "completed_group_5", "mean": 0.07073447108268738, "median": 0.07023533433675766, "p05": 0.04956750199198723, "p95": 0.09379896521568298 }, { "label": "current_group_partial", "mean": 0.5830056071281433, "median": 0.5862616896629333, "p05": 0.4658474922180176, "p95": 0.6849814057350159 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.7459560632705688, "group": 6, "latest_source_mass": 0.37691548466682434, "layer": 23, "max_source_mass": 0.37691548466682434, "mixer_index": 44, "normalized_entropy": 0.897243926765161, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.14318783581256866, "median": 0.1493408977985382, "p05": 0.08292607218027115, "p95": 0.19464610517024994 }, { "label": "completed_group_1", "mean": 0.1094653457403183, "median": 0.10858799517154694, "p05": 0.09432064741849899, "p95": 0.1279931515455246 }, { "label": "completed_group_2", "mean": 0.06349228322505951, "median": 0.06325754523277283, "p05": 0.05439673364162445, "p95": 0.0737837627530098 }, { "label": "completed_group_3", "mean": 0.07859592139720917, "median": 0.0775163471698761, "p05": 0.06603655964136124, "p95": 0.09357087314128876 }, { "label": "completed_group_4", "mean": 0.09567482024431229, "median": 0.09468592703342438, "p05": 0.07671141624450684, "p95": 0.11774884164333344 }, { "label": "completed_group_5", "mean": 0.13266831636428833, "median": 0.13199329376220703, "p05": 0.11244359612464905, "p95": 0.15504513680934906 }, { "label": "current_group_partial", "mean": 0.37691548466682434, "median": 0.37180638313293457, "p05": 0.31207239627838135, "p95": 0.4595233201980591 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.4055393934249878, "group": 6, "latest_source_mass": 0.5772827863693237, "layer": 23, "max_source_mass": 0.5772827863693237, "mixer_index": 45, "normalized_entropy": 0.7223043644164862, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.10154955089092255, "median": 0.08766824007034302, "p05": 0.04901552200317383, "p95": 0.20287738740444183 }, { "label": "completed_group_1", "mean": 0.05619276314973831, "median": 0.055477917194366455, "p05": 0.041608311235904694, "p95": 0.07371112704277039 }, { "label": "completed_group_2", "mean": 0.07791627198457718, "median": 0.07736755907535553, "p05": 0.055920060724020004, "p95": 0.10158214718103409 }, { "label": "completed_group_3", "mean": 0.053915880620479584, "median": 0.05215863138437271, "p05": 0.03798118978738785, "p95": 0.07156119495630264 }, { "label": "completed_group_4", "mean": 0.06398355215787888, "median": 0.06229078769683838, "p05": 0.040616974234580994, "p95": 0.09270931780338287 }, { "label": "completed_group_5", "mean": 0.06915920227766037, "median": 0.0684593990445137, "p05": 0.046986356377601624, "p95": 0.09408729523420334 }, { "label": "current_group_partial", "mean": 0.5772827863693237, "median": 0.5824689269065857, "p05": 0.4518115520477295, "p95": 0.6926202178001404 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.7853620052337646, "group": 6, "latest_source_mass": 0.3407441973686218, "layer": 24, "max_source_mass": 0.3407441973686218, "mixer_index": 46, "normalized_entropy": 0.9174945750195659, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.15284113585948944, "median": 0.1593906283378601, "p05": 0.08678612112998962, "p95": 0.211342453956604 }, { "label": "completed_group_1", "mean": 0.12206703424453735, "median": 0.12153112143278122, "p05": 0.1060115247964859, "p95": 0.1399766057729721 }, { "label": "completed_group_2", "mean": 0.06508444249629974, "median": 0.06482555717229843, "p05": 0.055350616574287415, "p95": 0.07557765394449234 }, { "label": "completed_group_3", "mean": 0.08347313106060028, "median": 0.08190013468265533, "p05": 0.06966532766819, "p95": 0.10050469636917114 }, { "label": "completed_group_4", "mean": 0.09811970591545105, "median": 0.09664031863212585, "p05": 0.07688035070896149, "p95": 0.12369741499423981 }, { "label": "completed_group_5", "mean": 0.13767032325267792, "median": 0.1373325139284134, "p05": 0.11642646044492722, "p95": 0.1606694608926773 }, { "label": "current_group_partial", "mean": 0.3407441973686218, "median": 0.33805108070373535, "p05": 0.271301805973053, "p95": 0.41781002283096313 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.509842872619629, "group": 6, "latest_source_mass": 0.5227814316749573, "layer": 24, "max_source_mass": 0.5227814316749573, "mixer_index": 47, "normalized_entropy": 0.7759057494780099, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.11788781732320786, "median": 0.10097194463014603, "p05": 0.06265684217214584, "p95": 0.2408325970172882 }, { "label": "completed_group_1", "mean": 0.06822571158409119, "median": 0.0673636943101883, "p05": 0.05209728330373764, "p95": 0.08648915588855743 }, { "label": "completed_group_2", "mean": 0.08556114137172699, "median": 0.08463940769433975, "p05": 0.06234566867351532, "p95": 0.1125786080956459 }, { "label": "completed_group_3", "mean": 0.06067090481519699, "median": 0.058974385261535645, "p05": 0.043267179280519485, "p95": 0.07997901737689972 }, { "label": "completed_group_4", "mean": 0.07228855788707733, "median": 0.06972275674343109, "p05": 0.04689513146877289, "p95": 0.106512151658535 }, { "label": "completed_group_5", "mean": 0.07258446514606476, "median": 0.07082539796829224, "p05": 0.050949547439813614, "p95": 0.09970614314079285 }, { "label": "current_group_partial", "mean": 0.5227814316749573, "median": 0.5290273427963257, "p05": 0.38773614168167114, "p95": 0.6392589807510376 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.839367389678955, "group": 7, "latest_source_mass": 0.29583215713500977, "layer": 25, "max_source_mass": 0.29583215713500977, "mixer_index": 48, "normalized_entropy": 0.9452478525649903, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.15906387567520142, "median": 0.15846745669841766, "p05": 0.09519976377487183, "p95": 0.24517810344696045 }, { "label": "completed_group_1", "mean": 0.13139380514621735, "median": 0.1308426856994629, "p05": 0.11478274315595627, "p95": 0.15000829100608826 }, { "label": "completed_group_2", "mean": 0.07717745751142502, "median": 0.07715500891208649, "p05": 0.06616044044494629, "p95": 0.08791667222976685 }, { "label": "completed_group_3", "mean": 0.09607303142547607, "median": 0.09436552226543427, "p05": 0.08152313530445099, "p95": 0.11384639143943787 }, { "label": "completed_group_4", "mean": 0.1107850894331932, "median": 0.1089654266834259, "p05": 0.08814431726932526, "p95": 0.139452263712883 }, { "label": "completed_group_5", "mean": 0.12967455387115479, "median": 0.1299237608909607, "p05": 0.10832332074642181, "p95": 0.15138982236385345 }, { "label": "completed_group_6", "mean": 0.29583215713500977, "median": 0.29647016525268555, "p05": 0.2276897132396698, "p95": 0.3604946434497833 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.5841834545135498, "group": 7, "latest_source_mass": 0.40767741203308105, "layer": 25, "max_source_mass": 0.40767741203308105, "mixer_index": 49, "normalized_entropy": 0.7618312045616817, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.07896879315376282, "median": 0.06809917092323303, "p05": 0.025932887569069862, "p95": 0.19368338584899902 }, { "label": "completed_group_1", "mean": 0.048752427101135254, "median": 0.04767637699842453, "p05": 0.034443892538547516, "p95": 0.0667153000831604 }, { "label": "completed_group_2", "mean": 0.055707164108753204, "median": 0.0549345538020134, "p05": 0.038696639239788055, "p95": 0.0750277042388916 }, { "label": "completed_group_3", "mean": 0.04139826446771622, "median": 0.03878612071275711, "p05": 0.026673054322600365, "p95": 0.05746251344680786 }, { "label": "completed_group_4", "mean": 0.04449018836021423, "median": 0.04196460545063019, "p05": 0.026162121444940567, "p95": 0.07164917886257172 }, { "label": "completed_group_5", "mean": 0.04285799711942673, "median": 0.04109475016593933, "p05": 0.029393533244729042, "p95": 0.06277710199356079 }, { "label": "completed_group_6", "mean": 0.2801477909088135, "median": 0.2763082981109619, "p05": 0.1455708146095276, "p95": 0.4254714846611023 }, { "label": "current_group_partial", "mean": 0.40767741203308105, "median": 0.4120897948741913, "p05": 0.21399420499801636, "p95": 0.5853031277656555 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.8125314712524414, "group": 7, "latest_source_mass": 0.3700707256793976, "layer": 26, "max_source_mass": 0.3700707256793976, "mixer_index": 50, "normalized_entropy": 0.8716433883436914, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.10889513790607452, "median": 0.11333495378494263, "p05": 0.04850132763385773, "p95": 0.1620434820652008 }, { "label": "completed_group_1", "mean": 0.08656036853790283, "median": 0.08640051633119583, "p05": 0.0650845319032669, "p95": 0.1084962785243988 }, { "label": "completed_group_2", "mean": 0.048123255372047424, "median": 0.04800213500857353, "p05": 0.03576180338859558, "p95": 0.06107594817876816 }, { "label": "completed_group_3", "mean": 0.06561170518398285, "median": 0.06379580497741699, "p05": 0.050968848168849945, "p95": 0.08332888036966324 }, { "label": "completed_group_4", "mean": 0.06917259097099304, "median": 0.06724178791046143, "p05": 0.050207968801259995, "p95": 0.09373342245817184 }, { "label": "completed_group_5", "mean": 0.08286689221858978, "median": 0.08183546364307404, "p05": 0.06535022705793381, "p95": 0.10340933501720428 }, { "label": "completed_group_6", "mean": 0.16869932413101196, "median": 0.16509854793548584, "p05": 0.11059704422950745, "p95": 0.23654375970363617 }, { "label": "current_group_partial", "mean": 0.3700707256793976, "median": 0.3694133758544922, "p05": 0.23441238701343536, "p95": 0.5025027990341187 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.50089693069458, "group": 7, "latest_source_mass": 0.49096518754959106, "layer": 26, "max_source_mass": 0.49096518754959106, "mixer_index": 51, "normalized_entropy": 0.7217788529328457, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.07974998652935028, "median": 0.07402504235506058, "p05": 0.04579866677522659, "p95": 0.1359037309885025 }, { "label": "completed_group_1", "mean": 0.04166398197412491, "median": 0.04038450866937637, "p05": 0.027992289513349533, "p95": 0.05969971418380737 }, { "label": "completed_group_2", "mean": 0.04607272148132324, "median": 0.044438350945711136, "p05": 0.030038591474294662, "p95": 0.06808309257030487 }, { "label": "completed_group_3", "mean": 0.03586764633655548, "median": 0.03317485377192497, "p05": 0.02203454077243805, "p95": 0.05572832003235817 }, { "label": "completed_group_4", "mean": 0.04223340004682541, "median": 0.03930673003196716, "p05": 0.02465122565627098, "p95": 0.06968094408512115 }, { "label": "completed_group_5", "mean": 0.04095568507909775, "median": 0.03919431194663048, "p05": 0.026239857077598572, "p95": 0.06058088690042496 }, { "label": "completed_group_6", "mean": 0.22249138355255127, "median": 0.2125401496887207, "p05": 0.10647299885749817, "p95": 0.3711881935596466 }, { "label": "current_group_partial", "mean": 0.49096518754959106, "median": 0.4988659620285034, "p05": 0.2878011167049408, "p95": 0.6676807403564453 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.8059929609298706, "group": 7, "latest_source_mass": 0.37139976024627686, "layer": 27, "max_source_mass": 0.37139976024627686, "mixer_index": 52, "normalized_entropy": 0.8684990295379666, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.10711923241615295, "median": 0.10972198843955994, "p05": 0.04896187782287598, "p95": 0.16201043128967285 }, { "label": "completed_group_1", "mean": 0.07651707530021667, "median": 0.07666032761335373, "p05": 0.05141128972172737, "p95": 0.10152198374271393 }, { "label": "completed_group_2", "mean": 0.058284737169742584, "median": 0.05812416970729828, "p05": 0.039273619651794434, "p95": 0.07745087146759033 }, { "label": "completed_group_3", "mean": 0.06609587371349335, "median": 0.06383530795574188, "p05": 0.04481930285692215, "p95": 0.09336265176534653 }, { "label": "completed_group_4", "mean": 0.07139429450035095, "median": 0.0694514587521553, "p05": 0.0485839769244194, "p95": 0.10034126043319702 }, { "label": "completed_group_5", "mean": 0.07493151724338531, "median": 0.07319007813930511, "p05": 0.051876723766326904, "p95": 0.10402432829141617 }, { "label": "completed_group_6", "mean": 0.17425751686096191, "median": 0.16937679052352905, "p05": 0.10449051856994629, "p95": 0.2624811828136444 }, { "label": "current_group_partial", "mean": 0.37139976024627686, "median": 0.3732905089855194, "p05": 0.23830299079418182, "p95": 0.5014542937278748 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.5622732639312744, "group": 7, "latest_source_mass": 0.4768889844417572, "layer": 27, "max_source_mass": 0.4768889844417572, "mixer_index": 53, "normalized_entropy": 0.7512946301290215, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.08775827288627625, "median": 0.08504067361354828, "p05": 0.0428270623087883, "p95": 0.144285649061203 }, { "label": "completed_group_1", "mean": 0.047765426337718964, "median": 0.04646281152963638, "p05": 0.031814731657505035, "p95": 0.06807165592908859 }, { "label": "completed_group_2", "mean": 0.053928617388010025, "median": 0.052262187004089355, "p05": 0.035437777638435364, "p95": 0.0784272700548172 }, { "label": "completed_group_3", "mean": 0.04293380305171013, "median": 0.03996016085147858, "p05": 0.02556057646870613, "p95": 0.06623870134353638 }, { "label": "completed_group_4", "mean": 0.04174989461898804, "median": 0.03903091698884964, "p05": 0.025013400241732597, "p95": 0.06869364529848099 }, { "label": "completed_group_5", "mean": 0.04734005779027939, "median": 0.04609068110585213, "p05": 0.029884062707424164, "p95": 0.06911227852106094 }, { "label": "completed_group_6", "mean": 0.20163494348526, "median": 0.19222964346408844, "p05": 0.09963390231132507, "p95": 0.3307305574417114 }, { "label": "current_group_partial", "mean": 0.4768889844417572, "median": 0.48383551836013794, "p05": 0.276212215423584, "p95": 0.6487575173377991 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.8757455348968506, "group": 7, "latest_source_mass": 0.3109493851661682, "layer": 28, "max_source_mass": 0.3109493851661682, "mixer_index": 54, "normalized_entropy": 0.9020429270551009, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.11563669890165329, "median": 0.12087491154670715, "p05": 0.05733206868171692, "p95": 0.1688953936100006 }, { "label": "completed_group_1", "mean": 0.08108697831630707, "median": 0.08185897022485733, "p05": 0.05175650864839554, "p95": 0.10753051191568375 }, { "label": "completed_group_2", "mean": 0.07280858606100082, "median": 0.07357072830200195, "p05": 0.04870978742837906, "p95": 0.0948689877986908 }, { "label": "completed_group_3", "mean": 0.07553227245807648, "median": 0.07362619787454605, "p05": 0.056317005306482315, "p95": 0.09756095707416534 }, { "label": "completed_group_4", "mean": 0.07606014609336853, "median": 0.07456053793430328, "p05": 0.05528895929455757, "p95": 0.1022825762629509 }, { "label": "completed_group_5", "mean": 0.06935927271842957, "median": 0.06843917071819305, "p05": 0.04978285729885101, "p95": 0.09167985618114471 }, { "label": "completed_group_6", "mean": 0.19856664538383484, "median": 0.19508783519268036, "p05": 0.1281973123550415, "p95": 0.27796900272369385 }, { "label": "current_group_partial", "mean": 0.3109493851661682, "median": 0.3105033040046692, "p05": 0.18172834813594818, "p95": 0.43916669487953186 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.6924852132797241, "group": 7, "latest_source_mass": 0.3315581679344177, "layer": 28, "max_source_mass": 0.3315581679344177, "mixer_index": 55, "normalized_entropy": 0.8139133413255193, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.08638577908277512, "median": 0.058608733117580414, "p05": 0.026544082909822464, "p95": 0.2740764319896698 }, { "label": "completed_group_1", "mean": 0.059912845492362976, "median": 0.05901291221380234, "p05": 0.03629481792449951, "p95": 0.08707985281944275 }, { "label": "completed_group_2", "mean": 0.06609142571687698, "median": 0.0647939145565033, "p05": 0.038691695779561996, "p95": 0.09703366458415985 }, { "label": "completed_group_3", "mean": 0.056745611131191254, "median": 0.05418796092271805, "p05": 0.032174739986658096, "p95": 0.08370660990476608 }, { "label": "completed_group_4", "mean": 0.06277773529291153, "median": 0.0600711889564991, "p05": 0.03501531109213829, "p95": 0.10094858705997467 }, { "label": "completed_group_5", "mean": 0.05352145433425903, "median": 0.05179818347096443, "p05": 0.032271772623062134, "p95": 0.08084158599376678 }, { "label": "completed_group_6", "mean": 0.2830069661140442, "median": 0.2759752869606018, "p05": 0.12981083989143372, "p95": 0.46204638481140137 }, { "label": "current_group_partial", "mean": 0.3315581679344177, "median": 0.3204721510410309, "p05": 0.13446375727653503, "p95": 0.5667697787284851 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.8980522155761719, "group": 8, "latest_source_mass": 0.20123262703418732, "layer": 29, "max_source_mass": 0.26602888107299805, "mixer_index": 56, "normalized_entropy": 0.9127701729200177, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.11603894829750061, "median": 0.09991352260112762, "p05": 0.030035395175218582, "p95": 0.2941743731498718 }, { "label": "completed_group_1", "mean": 0.08804173022508621, "median": 0.08880250155925751, "p05": 0.05664174258708954, "p95": 0.11779069900512695 }, { "label": "completed_group_2", "mean": 0.07975024729967117, "median": 0.07984748482704163, "p05": 0.05428915470838547, "p95": 0.10499341040849686 }, { "label": "completed_group_3", "mean": 0.09027000516653061, "median": 0.08898856490850449, "p05": 0.06189651042222977, "p95": 0.11996553838253021 }, { "label": "completed_group_4", "mean": 0.08430293202400208, "median": 0.0819794088602066, "p05": 0.05464346706867218, "p95": 0.12159879505634308 }, { "label": "completed_group_5", "mean": 0.07433462142944336, "median": 0.07287181913852692, "p05": 0.052520088851451874, "p95": 0.1010315865278244 }, { "label": "completed_group_6", "mean": 0.26602888107299805, "median": 0.2639757990837097, "p05": 0.15474572777748108, "p95": 0.3846370279788971 }, { "label": "completed_group_7", "mean": 0.20123262703418732, "median": 0.18140269815921783, "p05": 0.08419936150312424, "p95": 0.3782624900341034 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.7512003183364868, "group": 8, "latest_source_mass": 0.33482813835144043, "layer": 29, "max_source_mass": 0.33482813835144043, "mixer_index": 57, "normalized_entropy": 0.7970056117156376, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.07744461297988892, "median": 0.06970229744911194, "p05": 0.014401955530047417, "p95": 0.17144526541233063 }, { "label": "completed_group_1", "mean": 0.046086348593235016, "median": 0.04394831880927086, "p05": 0.028066720813512802, "p95": 0.0718703418970108 }, { "label": "completed_group_2", "mean": 0.05025433003902435, "median": 0.04861236736178398, "p05": 0.029644077643752098, "p95": 0.07761462777853012 }, { "label": "completed_group_3", "mean": 0.03748488053679466, "median": 0.03434944897890091, "p05": 0.020774798467755318, "p95": 0.05938909202814102 }, { "label": "completed_group_4", "mean": 0.04005061462521553, "median": 0.037498839199543, "p05": 0.019541850313544273, "p95": 0.06922487914562225 }, { "label": "completed_group_5", "mean": 0.041623204946517944, "median": 0.039552345871925354, "p05": 0.022081367671489716, "p95": 0.06821337342262268 }, { "label": "completed_group_6", "mean": 0.198423832654953, "median": 0.18318486213684082, "p05": 0.07548617571592331, "p95": 0.3721975088119507 }, { "label": "completed_group_7", "mean": 0.17380405962467194, "median": 0.1514919549226761, "p05": 0.05777408182621002, "p95": 0.36326974630355835 }, { "label": "current_group_partial", "mean": 0.33482813835144043, "median": 0.33602219820022583, "p05": 0.0795319601893425, "p95": 0.5979493856430054 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.0160694122314453, "group": 8, "latest_source_mass": 0.21711516380310059, "layer": 30, "max_source_mass": 0.21711516380310059, "mixer_index": 58, "normalized_entropy": 0.9175527313077866, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.09385271370410919, "median": 0.09211117774248123, "p05": 0.032377734780311584, "p95": 0.16496515274047852 }, { "label": "completed_group_1", "mean": 0.06208505108952522, "median": 0.06140740588307381, "p05": 0.0407925546169281, "p95": 0.0863042101264 }, { "label": "completed_group_2", "mean": 0.06560768187046051, "median": 0.06516212224960327, "p05": 0.041897669434547424, "p95": 0.09085705131292343 }, { "label": "completed_group_3", "mean": 0.07029052078723907, "median": 0.06780634075403214, "p05": 0.04846132919192314, "p95": 0.09917552024126053 }, { "label": "completed_group_4", "mean": 0.06935097277164459, "median": 0.06708167493343353, "p05": 0.04668639227747917, "p95": 0.09815650433301926 }, { "label": "completed_group_5", "mean": 0.056832700967788696, "median": 0.05595892667770386, "p05": 0.038822539150714874, "p95": 0.0771557092666626 }, { "label": "completed_group_6", "mean": 0.19425815343856812, "median": 0.19145764410495758, "p05": 0.12249615788459778, "p95": 0.27496030926704407 }, { "label": "completed_group_7", "mean": 0.17060703039169312, "median": 0.16317546367645264, "p05": 0.09239593893289566, "p95": 0.2726203501224518 }, { "label": "current_group_partial", "mean": 0.21711516380310059, "median": 0.21198657155036926, "p05": 0.11019522696733475, "p95": 0.3391069769859314 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 1.7837426662445068, "group": 8, "latest_source_mass": 0.29026859998703003, "layer": 30, "max_source_mass": 0.29026859998703003, "mixer_index": 59, "normalized_entropy": 0.8118162725118464, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.08836327493190765, "median": 0.07904411852359772, "p05": 0.02631576545536518, "p95": 0.17534467577934265 }, { "label": "completed_group_1", "mean": 0.04953558370471001, "median": 0.04658038914203644, "p05": 0.025181813165545464, "p95": 0.08318150043487549 }, { "label": "completed_group_2", "mean": 0.04813014343380928, "median": 0.045740917325019836, "p05": 0.026714935898780823, "p95": 0.07843323051929474 }, { "label": "completed_group_3", "mean": 0.0373263955116272, "median": 0.03401809185743332, "p05": 0.01990434154868126, "p95": 0.06290259957313538 }, { "label": "completed_group_4", "mean": 0.037015400826931, "median": 0.03405017405748367, "p05": 0.018646355718374252, "p95": 0.06544196605682373 }, { "label": "completed_group_5", "mean": 0.03390160948038101, "median": 0.03198609501123428, "p05": 0.018189728260040283, "p95": 0.05570102855563164 }, { "label": "completed_group_6", "mean": 0.1840541511774063, "median": 0.17327937483787537, "p05": 0.07966600358486176, "p95": 0.32982927560806274 }, { "label": "completed_group_7", "mean": 0.23140482604503632, "median": 0.22406929731369019, "p05": 0.08831030130386353, "p95": 0.40350067615509033 }, { "label": "current_group_partial", "mean": 0.29026859998703003, "median": 0.2745037376880646, "p05": 0.10280418395996094, "p95": 0.5399085879325867 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 1.9973840713500977, "group": 8, "latest_source_mass": 0.19101998209953308, "layer": 31, "max_source_mass": 0.20831164717674255, "mixer_index": 60, "normalized_entropy": 0.9090486661912383, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.10261696577072144, "median": 0.10389091819524765, "p05": 0.01371697522699833, "p95": 0.21163305640220642 }, { "label": "completed_group_1", "mean": 0.06039813905954361, "median": 0.06085003539919853, "p05": 0.026107093319296837, "p95": 0.09348798543214798 }, { "label": "completed_group_2", "mean": 0.06671924144029617, "median": 0.06656201928853989, "p05": 0.034684643149375916, "p95": 0.09819839894771576 }, { "label": "completed_group_3", "mean": 0.08377155661582947, "median": 0.0804532840847969, "p05": 0.05448959022760391, "p95": 0.12039553374052048 }, { "label": "completed_group_4", "mean": 0.07002593576908112, "median": 0.06737451255321503, "p05": 0.044841229915618896, "p95": 0.10655929893255234 }, { "label": "completed_group_5", "mean": 0.05768897384405136, "median": 0.056077782064676285, "p05": 0.039495617151260376, "p95": 0.08242568373680115 }, { "label": "completed_group_6", "mean": 0.20831164717674255, "median": 0.2021285891532898, "p05": 0.11652229726314545, "p95": 0.31706833839416504 }, { "label": "completed_group_7", "mean": 0.1594475358724594, "median": 0.1546279788017273, "p05": 0.06593266129493713, "p95": 0.2735573351383209 }, { "label": "current_group_partial", "mean": 0.19101998209953308, "median": 0.1713225245475769, "p05": 0.06408411264419556, "p95": 0.3850628137588501 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 1.969987154006958, "group": 8, "latest_source_mass": 0.16864418983459473, "layer": 31, "max_source_mass": 0.24503237009048462, "mixer_index": 61, "normalized_entropy": 0.8965797917640489, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.08120635151863098, "median": 0.061164144426584244, "p05": 0.0259944386780262, "p95": 0.23622018098831177 }, { "label": "completed_group_1", "mean": 0.07894632965326309, "median": 0.07658770680427551, "p05": 0.0496862456202507, "p95": 0.11596276611089706 }, { "label": "completed_group_2", "mean": 0.08615730702877045, "median": 0.08467075228691101, "p05": 0.050409119576215744, "p95": 0.12811316549777985 }, { "label": "completed_group_3", "mean": 0.07609779387712479, "median": 0.0722871869802475, "p05": 0.04099677503108978, "p95": 0.11950768530368805 }, { "label": "completed_group_4", "mean": 0.058728985488414764, "median": 0.056080836802721024, "p05": 0.03323862701654434, "p95": 0.09344407916069031 }, { "label": "completed_group_5", "mean": 0.05076971277594566, "median": 0.04946017265319824, "p05": 0.03047419898211956, "p95": 0.07563400268554688 }, { "label": "completed_group_6", "mean": 0.24503237009048462, "median": 0.24110376834869385, "p05": 0.10613486170768738, "p95": 0.3991503119468689 }, { "label": "completed_group_7", "mean": 0.1544169783592224, "median": 0.1400495171546936, "p05": 0.06199025735259056, "p95": 0.2995939254760742 }, { "label": "current_group_partial", "mean": 0.16864418983459473, "median": 0.15222924947738647, "p05": 0.0543956533074379, "p95": 0.3329208791255951 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.071648359298706, "group": 8, "latest_source_mass": 0.11520501971244812, "layer": 32, "max_source_mass": 0.20955611765384674, "mixer_index": 62, "normalized_entropy": 0.9428478002054053, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.10569609701633453, "median": 0.08689188957214355, "p05": 0.03173588588833809, "p95": 0.24571751058101654 }, { "label": "completed_group_1", "mean": 0.09922641515731812, "median": 0.09795533865690231, "p05": 0.06407676637172699, "p95": 0.14120742678642273 }, { "label": "completed_group_2", "mean": 0.08180510997772217, "median": 0.080217644572258, "p05": 0.054607540369033813, "p95": 0.11442367732524872 }, { "label": "completed_group_3", "mean": 0.10646799206733704, "median": 0.10510502755641937, "p05": 0.07415887713432312, "p95": 0.14216329157352448 }, { "label": "completed_group_4", "mean": 0.0826798751950264, "median": 0.08022409677505493, "p05": 0.05491747707128525, "p95": 0.11860857903957367 }, { "label": "completed_group_5", "mean": 0.06615597754716873, "median": 0.06485182791948318, "p05": 0.04477996751666069, "p95": 0.09238749742507935 }, { "label": "completed_group_6", "mean": 0.20955611765384674, "median": 0.206287682056427, "p05": 0.129981130361557, "p95": 0.2996031939983368 }, { "label": "completed_group_7", "mean": 0.13320739567279816, "median": 0.12465103715658188, "p05": 0.06303662061691284, "p95": 0.23065781593322754 }, { "label": "current_group_partial", "mean": 0.11520501971244812, "median": 0.10386130958795547, "p05": 0.03912060335278511, "p95": 0.22735844552516937 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 1.9781701564788818, "group": 8, "latest_source_mass": 0.16112206876277924, "layer": 32, "max_source_mass": 0.22251944243907928, "mixer_index": 63, "normalized_entropy": 0.9003040366848136, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.09446848928928375, "median": 0.074150450527668, "p05": 0.042592622339725494, "p95": 0.19443637132644653 }, { "label": "completed_group_1", "mean": 0.07056106626987457, "median": 0.06868506222963333, "p05": 0.04818308353424072, "p95": 0.0982733890414238 }, { "label": "completed_group_2", "mean": 0.08033448457717896, "median": 0.08047192543745041, "p05": 0.048296794295310974, "p95": 0.11253741383552551 }, { "label": "completed_group_3", "mean": 0.0658978745341301, "median": 0.06256154924631119, "p05": 0.04223886504769325, "p95": 0.0967884361743927 }, { "label": "completed_group_4", "mean": 0.058234501630067825, "median": 0.0563921295106411, "p05": 0.03529864549636841, "p95": 0.08846980333328247 }, { "label": "completed_group_5", "mean": 0.04316519945859909, "median": 0.042268507182598114, "p05": 0.029443470761179924, "p95": 0.05996086448431015 }, { "label": "completed_group_6", "mean": 0.2036968618631363, "median": 0.19645483791828156, "p05": 0.09398645162582397, "p95": 0.3395216464996338 }, { "label": "completed_group_7", "mean": 0.22251944243907928, "median": 0.2130982130765915, "p05": 0.11867167800664902, "p95": 0.35465875267982483 }, { "label": "current_group_partial", "mean": 0.16112206876277924, "median": 0.14023703336715698, "p05": 0.06008770316839218, "p95": 0.3247341811656952 } ], "sources": 9 }, { "branch": "output", "entropy_mean": 1.6036605834960938, "group": null, "latest_source_mass": 0.31639498472213745, "layer": null, "max_source_mass": 0.3412485122680664, "mixer_index": null, "normalized_entropy": 0.7298573846467136, "offset": null, "source_summaries": [ { "label": "embedding", "mean": 0.09759747982025146, "median": 0.09915056824684143, "p05": 0.025461740791797638, "p95": 0.16398483514785767 }, { "label": "completed_group_1", "mean": 0.04215940088033676, "median": 0.04050712287425995, "p05": 0.025720715522766113, "p95": 0.06398092210292816 }, { "label": "completed_group_2", "mean": 0.041145481169223785, "median": 0.03881259262561798, "p05": 0.023510465398430824, "p95": 0.0662468671798706 }, { "label": "completed_group_3", "mean": 0.029554028064012527, "median": 0.027420252561569214, "p05": 0.01627083122730255, "p95": 0.04924973472952843 }, { "label": "completed_group_4", "mean": 0.0289996936917305, "median": 0.027226522564888, "p05": 0.01560397818684578, "p95": 0.048351090401411057 }, { "label": "completed_group_5", "mean": 0.03647579252719879, "median": 0.0347352996468544, "p05": 0.020744025707244873, "p95": 0.058257561177015305 }, { "label": "completed_group_6", "mean": 0.06642461568117142, "median": 0.057809896767139435, "p05": 0.022837381809949875, "p95": 0.14078694581985474 }, { "label": "completed_group_7", "mean": 0.3412485122680664, "median": 0.3336990475654602, "p05": 0.0819513276219368, "p95": 0.6423859596252441 }, { "label": "completed_group_8", "mean": 0.31639498472213745, "median": 0.30713334679603577, "p05": 0.05841828137636185, "p95": 0.6156357526779175 } ], "sources": 9 } ], "mode": "detached_learned", "optimizer_state_after": "ed03e6fbd4a12d8b063dcb22e0437754285f54d585374cd52fbd534f05d24637", "optimizer_state_before": "ed03e6fbd4a12d8b063dcb22e0437754285f54d585374cd52fbd534f05d24637", "optimizer_state_unchanged": true, "positions": { "attention_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "5e6ecbcd78543fc42d4098a51169dca446d404ce384015643e84d2c78433c949", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.0933690071105957, 0.24537226557731628, 0.07698935270309448, 0.0523640513420105, 0.09380833059549332, 0.03647654876112938, 0.039120711386203766, 0.06371714919805527, 0.05156813934445381, 0.03728010877966881, 0.03983496502041817, 0.040284499526023865, 0.046871740370988846, 0.04314620420336723, 0.04033593460917473, 0.04471700266003609, 0.08397205173969269, 0.0528092160820961, 0.04443390294909477, 0.07560808956623077, 0.03708760067820549, 0.03599191829562187, 0.033304885029792786, 0.04061548039317131, 0.025487789884209633, 0.021565141156315804, 0.02342291921377182, 0.017244853079319, 0.024166326969861984, 0.016819356009364128, 0.021590016782283783, 0.026201345026493073 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.4657019391961512e-05, "first_to_last_ratio": 0.6094218601945491, "imbalance_abs_log_ratio": 0.4952445414057865, "last_quartile_mean": 2.4050695174082648e-05, "mean": 2.0123495630741672e-05, "normalized": [ 0.6757922451357102, 0.7862872769192628, 0.6956720089134658, 0.6457028717597475, 0.8486081172555344, 0.7866590564328257, 0.7355608412572463, 0.6525459469054529, 0.5676581895263636, 0.5065196225058339, 0.47654698952417385, 0.4567643060278592, 0.6388028952758977, 0.6012584522984605, 0.5677051478189228, 0.5320958926863361, 0.6601484899512206, 0.6199678744151763, 0.6252065487643861, 0.6703348689792878, 3.1476439082057013, 2.6843722856032843, 2.147025515739066, 1.709881143629556, 2.310023308278431, 2.0253173950794507, 1.5260326037153789, 0.9852764539113186, 1.0403760218043532, 0.7424001651630263, 0.4909889920627068, 0.44082456445456286 ], "peak_layer": 21, "peak_normalized": 3.1476439082057013, "population_cv": 0.7023423019155828, "rest_mean": 1.4907078501117346e-05, "spike_contrast": 3.2395447657361793, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.829214813071303e-05, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 1.359930229227757e-05, 1.5822848581592552e-05, 1.3999352631799411e-05, 1.299379891861463e-05, 1.7076961739803664e-05, 1.5830330085009336e-05, 1.4802055375184864e-05, 1.3131505511410069e-05, 1.1423267096688505e-05, 1.0192945410381071e-05, 9.58979126153281e-06, 9.191694516630378e-06, 1.2854947271989658e-05, 1.2099421837774571e-05, 1.1424212061683647e-05, 1.0707629371609073e-05, 1.32844952531741e-05, 1.2475920811994001e-05, 1.2581341252371203e-05, 1.348948080703849e-05, 6.334159843390808e-05, 5.401895396062173e-05, 4.3205658585065976e-05, 3.4408785722916946e-05, 4.648574395105243e-05, 4.075646575074643e-05, 3.0709110433235765e-05, 1.9827206415357068e-05, 2.0936002329108305e-05, 1.4939686479920056e-05, 9.880414836516138e-06, 8.870931196724996e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 5.829688234371133e-05, "first_to_last_ratio": 0.6072503732166264, "imbalance_abs_log_ratio": 0.4988140964987654, "last_quartile_mean": 9.600139401300112e-05, "mean": 8.040016996346822e-05, "normalized": [ 0.6783743745296411, 0.7863508052165757, 0.7000215696280339, 0.647124428880743, 0.8353484843612504, 0.7743411566859596, 0.7293747929726763, 0.6497369355766927, 0.5707307458024328, 0.510545581500116, 0.4799754930157162, 0.4590012320770852, 0.6477982227600472, 0.6029757489491335, 0.5684984161378385, 0.5312301622013348, 0.6575876681964574, 0.6204772859169876, 0.6292313575338936, 0.6709435017853759, 3.152621969125565, 2.693161601564409, 2.150493956812164, 1.7016971884685128, 2.3039790532500866, 2.0188035652467615, 1.519776732418517, 0.9811964658615112, 1.039948884922119, 0.7451655332994308, 0.496455864160715, 0.4470312211422162 ], "peak_layer": 21, "peak_normalized": 3.152621969125565, "population_cv": 0.7020159094649734, "rest_mean": 5.954986355155362e-05, "spike_contrast": 3.24084411076324, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00019299182458780705, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 5.454141501104459e-05, 6.322273839032277e-05, 5.6281853176187724e-05, 5.2028914069524035e-05, 6.716216012137011e-05, 6.225716060725972e-05, 5.864185732207261e-05, 5.223896005190909e-05, 4.588684896589257e-05, 4.104795152670704e-05, 3.8590111216763034e-05, 3.690377707243897e-05, 5.208308721194044e-05, 4.847935269935988e-05, 4.57073692814447e-05, 4.27109953307081e-05, 5.287016028887592e-05, 4.988647924619727e-05, 5.059030809206888e-05, 5.394397157942876e-05, 0.0002534713421482593, 0.00021653065050486475, 0.00017290007963310927, 0.00013681674317922443, 0.00018524030747357756, 0.00016231214976869524, 0.0001221903075929731, 7.888836262281984e-05, 8.361206710105762e-05, 5.991143552819267e-05, 3.991513585788198e-05, 3.594138615881093e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.05169825907796621, "first_to_last_ratio": 0.607250394007124, "imbalance_abs_log_ratio": 0.4988140622616565, "last_quartile_mean": 0.08513499470427632, "mean": 0.07129967457149178, "normalized": [ 0.6783743842436116, 0.7863508228047997, 0.700021554566655, 0.6471244375634602, 0.8353485410067275, 0.7743411217427667, 0.7293748339206622, 0.6497369595226863, 0.5707307154188002, 0.5105455806753446, 0.47997548951564134, 0.4590012256825957, 0.6477982322753953, 0.6029757319104283, 0.5684984046970705, 0.5312301796601451, 0.6575876919843755, 0.6204773092105361, 0.6292313633824181, 0.6709435195164998, 3.152621923151655, 2.693161643472544, 2.150493986671358, 1.7016971770874902, 2.30397915541657, 2.0188034853391237, 1.5197766802362542, 0.9811964117969386, 1.0399488445555949, 0.7451654867762114, 0.49645587282964704, 0.44703123336599304 ], "peak_layer": 21, "peak_normalized": 3.152621923151655, "population_cv": 0.7020159090531873, "rest_mean": 0.052809414074376775, "spike_contrast": 3.2408441611351573, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.17114708125591277, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 0.04836787283420563, 0.05606655776500702, 0.04991130903363228, 0.04613976180553436, 0.059560079127550125, 0.05521026998758316, 0.05200418829917908, 0.046326033771038055, 0.04069291427731514, 0.03640173375606537, 0.03422209620475769, 0.03272663801908493, 0.04618780314922333, 0.04299197345972061, 0.040533751249313354, 0.037876538932323456, 0.046885788440704346, 0.0442398302257061, 0.0448639914393425, 0.04783805459737778, 0.22478091716766357, 0.19202154874801636, 0.1533295214176178, 0.12133045494556427, 0.1642729640007019, 0.1439400315284729, 0.10835958272218704, 0.06995898485183716, 0.0741480141878128, 0.05313005670905113, 0.03539714217185974, 0.0318731814622879 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 2.317203296797743e-06, "first_to_last_ratio": 0.35664923571220175, "imbalance_abs_log_ratio": 1.0310025132235103, "last_quartile_mean": 6.49714920086808e-06, "mean": 4.492464356076198e-06, "normalized": [ 0.5694178936246215, 0.527096091999763, 0.5069565208451643, 0.5168980803173514, 0.5167362729922896, 0.5011287746943238, 0.4906181312174465, 0.49753075069099545, 0.4121289001968579, 0.3863295396034041, 0.3787441822616284, 0.3878491467448858, 0.49538413383836966, 0.4937248622259753, 0.4965317157240282, 0.4922453146709141, 0.5531637227610273, 0.5731353625097597, 0.5949767701689571, 0.6269395119699129, 2.991445440442326, 2.8607213177085975, 2.466076440453951, 2.094358907038209, 2.7670265254785567, 2.47746135973099, 1.9032918373610526, 1.2284996183916332, 1.2122250507961028, 0.87482980171944, 0.5899517341136686, 0.5165762877077978 ], "peak_layer": 21, "peak_normalized": 2.991445440442326, "population_cv": 0.8381751118691627, "rest_mean": 3.131475834918631e-06, "spike_contrast": 3.7815403964740346, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.184180237032706e-05, "top_five_layers": [ 21, 22, 25, 26, 23 ], "top_five_spike_overlap": 4 }, "values": [ 2.5580895908206003e-06, 2.367960405535996e-06, 2.2774840999773005e-06, 2.322146201549913e-06, 2.321419287909521e-06, 2.2513031581183895e-06, 2.2040844669390935e-06, 2.23513916353113e-06, 1.8514743942432688e-06, 1.7355716863676207e-06, 1.7014947388815926e-06, 1.742398467285966e-06, 2.2254955638345564e-06, 2.2180413452588255e-06, 2.2306510345515562e-06, 2.2113945306045935e-06, 2.485068307578331e-06, 2.5747901872819057e-06, 2.67291193267738e-06, 2.8165034109406406e-06, 1.343896201433381e-05, 1.2851688552473206e-05, 1.107876050809864e-05, 9.408832738699857e-06, 1.2430768038029782e-05, 1.1129906852147542e-05, 8.550470738555305e-06, 5.518990747077623e-06, 5.44587783224415e-06, 3.930141701857792e-06, 2.6503371373109985e-06, 2.3207005597214447e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 4.480986444832524e-05, "first_to_last_ratio": 0.7542988096897616, "imbalance_abs_log_ratio": 0.28196669015672604, "last_quartile_mean": 5.940598589404544e-05, "mean": 5.624602624720865e-05, "normalized": [ 0.7593253658919704, 0.8584935198542845, 0.7758342247870141, 0.7257052209284447, 0.9115911301096014, 0.8478599752338415, 0.7912991525483561, 0.7033001562001767, 0.6342038809025199, 0.5708934677878957, 0.5360030772522035, 0.5119164360939763, 0.7122797103032084, 0.65763851786042, 0.6170495828015774, 0.5742094044317299, 0.7380398429228872, 0.6958734981683372, 0.7020379955965178, 0.7406705623157649, 3.198556354491257, 2.6580706472996987, 2.0667008648707084, 1.5629991771988188, 2.1587737798302467, 1.7912836314300635, 1.3377158632775958, 0.8975280712077295, 0.8407131217921131, 0.5976858891866252, 0.431587196610723, 0.394160680813692 ], "peak_layer": 21, "peak_normalized": 3.198556354491257, "population_cv": 0.6612253154131089, "rest_mean": 4.240304419740655e-05, "spike_contrast": 3.089356714727384, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013099812931613997, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 4.270903446013108e-05, 4.828684905078262e-05, 4.363759217085317e-05, 4.081803490407765e-05, 5.127337863086723e-05, 4.768875442096032e-05, 4.4507432903628796e-05, 3.955783904530108e-05, 3.567144813132472e-05, 3.211048897355795e-05, 3.0148043151712045e-05, 2.8793265300919302e-05, 4.006290328106843e-05, 3.698955333675258e-05, 3.4706587030086666e-05, 3.229699723306112e-05, 4.151180837652646e-05, 3.9140119042713195e-05, 3.948684752685949e-05, 4.16597758885473e-05, 0.00017990608466789126, 0.00014950591139495373, 0.00011624371109064668, 8.791249274509028e-05, 0.00012142244668211788, 0.00010075258614961058, 7.524120155721903e-05, 5.0482387450756505e-05, 4.7286772314691916e-05, 3.3617456210777164e-05, 2.4275064788525924e-05, 2.216997199866455e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 3.384813680895604e-05, "first_to_last_ratio": 0.9740578591651334, "imbalance_abs_log_ratio": 0.026284573443628244, "last_quartile_mean": 3.474961624760908e-05, "mean": 3.869269224310301e-05, "normalized": [ 0.8541900771208318, 0.9413918806879924, 0.859570413488977, 0.8162196112220202, 0.9909206111274242, 0.9266644039643532, 0.8522558488291101, 0.7571395238293102, 0.7081640210424396, 0.6442698940442247, 0.603596002806512, 0.575557073332959, 0.7887193829441419, 0.7247641884135786, 0.6750858213648586, 0.6336813762759561, 0.8435498470464944, 0.7939958245882761, 0.7970984689300282, 0.8220042438012486, 3.299009695161972, 2.613224785716232, 1.9509379481481457, 1.3432491513722262, 1.9254306181237832, 1.4942554786145408, 1.1183073895840108, 0.8225722329666332, 0.6567052933181462, 0.46462952270788027, 0.36224409286636666, 0.34059527655932637 ], "peak_layer": 21, "peak_normalized": 3.299009695161972, "population_cv": 0.6253906355726564, "rest_mean": 2.9905363724672633e-05, "spike_contrast": 2.880562397960604, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 8.614426624262706e-05, "top_five_layers": [ 21, 22, 23, 25, 26 ], "top_five_spike_overlap": 4 }, "values": [ 3.305091377114877e-05, 3.642498631961644e-05, 3.325909347040579e-05, 3.158173421979882e-05, 3.834138624370098e-05, 3.5855140595231205e-05, 3.297607327112928e-05, 2.9295766580617055e-05, 2.740077252383344e-05, 2.4928536731749773e-05, 2.3354754375759512e-05, 2.2269852706813253e-05, 3.051767635042779e-05, 2.804307769110892e-05, 2.612088792375289e-05, 2.4518838472431526e-05, 3.263921462348662e-05, 3.072183608310297e-05, 3.084188574575819e-05, 3.180555722792633e-05, 0.00012764756684191525, 0.00010111270239576697, 7.548704161308706e-05, 5.1973926019854844e-05, 7.450009434251115e-05, 5.781676736660302e-05, 4.327032365836203e-05, 3.182753425789997e-05, 2.5409695808775723e-05, 1.7977767129195854e-05, 1.4016199202160351e-05, 1.3178548215364572e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.0001188372980323038, "first_to_last_ratio": 0.585909807801185, "imbalance_abs_log_ratio": 0.534589412852804, "last_quartile_mean": 0.00020282524110371014, "mean": 0.00016756316449573205, "normalized": [ 0.6676704870164084, 0.7483084683488639, 0.6769676096236417, 0.6368287558643128, 0.8100066941936953, 0.7564679298832396, 0.7242231413041909, 0.6531986131482728, 0.5690738743162227, 0.5079946750459964, 0.47515152884923867, 0.45873382032473325, 0.6302192489862594, 0.5851397763517093, 0.5571686419883745, 0.5132471085839729, 0.6453685522588855, 0.614712668030142, 0.6206621143750464, 0.6587669430337698, 3.1211444947467313, 2.6936917077524942, 2.215351775191127, 1.776377347392615, 2.3967757154563434, 2.05632794722869, 1.5725614581941492, 1.0240120317645556, 1.032250695352248, 0.7057034519699571, 0.4689146276836353, 0.42697809574047824 ], "peak_layer": 21, "peak_normalized": 3.1211444947467313, "population_cv": 0.718404168086637, "rest_mean": 0.00012285891932185255, "spike_contrast": 3.3287456107546904, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0004089660884346813, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 0.00011187697964487597, 0.00012538893497548997, 0.00011343483492964879, 0.00010670904157450423, 0.0001357272849418223, 0.00012675616017077118, 0.00012135312135796994, 0.00010945202666334808, 9.535581921227276e-05, 8.512119529768825e-05, 7.961789378896356e-05, 7.686689059482887e-05, 0.0001056015316862613, 9.804787259781733e-05, 9.336094080936164e-05, 8.60013096826151e-05, 0.00010813999688252807, 0.00010300319991074502, 0.00010400010796729475, 0.00011038507363991812, 0.000522988848388195, 0.0004513635067269206, 0.0003712113539222628, 0.0002976554096676409, 0.0004016113234683871, 0.000344564818078652, 0.0002635033742990345, 0.00017158669652417302, 0.0001729671930661425, 0.00011824990360764787, 7.857281889300793e-05, 7.154580089263618e-05 ] } } }, "mlp_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "c0b8018c349b1bd09e5d17858eb312a2323da0e92de9651e2bb08ae4adf115c7", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.13583384454250336, 0.0415157787501812, 0.018140379339456558, 0.039518751204013824, 0.022877270355820656, 0.015532293356955051, 0.02314678393304348, 0.05432315170764923, 0.03743875026702881, 0.04006272554397583, 0.045583780854940414, 0.05995568633079529, 0.038651175796985626, 0.04559725895524025, 0.05632574483752251, 0.07952121645212173, 0.042337141931056976, 0.05044656619429588, 0.04418695345520973, 0.06310147047042847, 0.029010163620114326, 0.04586305841803551, 0.06033914536237717, 0.07656749337911606, 0.045884523540735245, 0.051604680716991425, 0.06502768397331238, 0.06792232394218445, 0.060306549072265625, 0.08025261014699936, 0.09663844108581543, 0.12230555713176727 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.4509995708067436e-05, "first_to_last_ratio": 0.7743031141222962, "imbalance_abs_log_ratio": 0.2557918617425859, "last_quartile_mean": 1.8739425740932347e-05, "mean": 1.7722179109114222e-05, "normalized": [ 0.7655666384169513, 0.8917483357831588, 0.7902102983812471, 0.7316333655654478, 0.9592978291511814, 0.8828085684394261, 0.8112146819967133, 0.7175029311175319, 0.593548092754179, 0.5514020456598471, 0.522638501525885, 0.5037614169375741, 0.7045039393419045, 0.6588959250307225, 0.6114268576480177, 0.579302599818883, 0.7435448955446501, 0.6887296875838754, 0.6921332531242588, 0.8093892960298869, 3.2325455196933226, 2.5123032009763855, 1.9971252826810495, 1.5895697479726458, 2.322878133149869, 1.7452609194969326, 1.1261946298991334, 0.8913309023592738, 0.8429252014113704, 0.5609036083922285, 0.5024807882604625, 0.46722290585598436 ], "peak_layer": 21, "peak_normalized": 3.2325455196933226, "population_cv": 0.6537085087930902, "rest_mean": 1.335436960895701e-05, "spike_contrast": 3.09324600258607, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.130835040996317e-05, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 1.3567509085987695e-05, 1.580372372700367e-05, 1.4004248441779055e-05, 1.2966137546754908e-05, 1.700084794720169e-05, 1.564529156894423e-05, 1.437649189028889e-05, 1.2715715456579346e-05, 1.05189656096627e-05, 9.772045814315788e-06, 9.2622931333608e-06, 8.927750059228856e-06, 1.2485344996093772e-05, 1.167707159765996e-05, 1.0835816283361055e-05, 1.0266504432365764e-05, 1.3177235814509913e-05, 1.2205790881125722e-05, 1.2266109479242004e-05, 1.4344142073241528e-05, 5.7287750678369775e-05, 4.452348730410449e-05, 3.539341196301393e-05, 2.8170639780000784e-05, 4.116646232432686e-05, 3.092982660746202e-05, 1.9958622942795046e-05, 1.579632589709945e-05, 1.4938471394998487e-05, 9.940434210875537e-06, 8.905054528440814e-06, 8.280208021460567e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 5.7723796544451034e-05, "first_to_last_ratio": 0.7709532556520535, "imbalance_abs_log_ratio": 0.2601275354642249, "last_quartile_mean": 7.487327684430056e-05, "mean": 7.085550987540046e-05, "normalized": [ 0.767459364760998, 0.8911005669273344, 0.7949447373790853, 0.7325731165004719, 0.9435545502986998, 0.8683878924182112, 0.8054854811328411, 0.7138473259026528, 0.5978031833741329, 0.5556350739964981, 0.526183189154655, 0.5057617239608265, 0.7151088387962502, 0.660810541031278, 0.6122576977156078, 0.5785452674352782, 0.739958743085955, 0.6892518806239643, 0.6970577547891826, 0.8113719765230756, 3.2405178430845596, 2.524955075289844, 1.989505542525185, 1.5842933275736053, 2.315101352895563, 1.736925913599177, 1.1174905309838439, 0.8878674587909959, 0.8457916652531124, 0.5666462793963278, 0.5093376021450956, 0.4744685026556913 ], "peak_layer": 21, "peak_normalized": 3.2405178430845596, "population_cv": 0.6532804064480114, "rest_mean": 5.3392583881591066e-05, "spike_contrast": 3.093225654863168, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00016515531024197116, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 5.437872459879145e-05, 6.313938501989469e-05, 5.63262146897614e-05, 5.190684169065207e-05, 6.685603875666857e-05, 6.153006688691676e-05, 5.7073084462899715e-05, 5.058001625002362e-05, 4.2357649363111705e-05, 3.936980647267774e-05, 3.728297815541737e-05, 3.58360048267059e-05, 5.066940138931386e-05, 4.682206781581044e-05, 4.33818313467782e-05, 4.099311991012655e-05, 5.2430154028115794e-05, 4.883729343418963e-05, 4.93903826281894e-05, 5.749017509515397e-05, 0.0002296085440320894, 0.00017890697927214205, 0.0001409674296155572, 0.00011225591151742265, 0.00016403768677264452, 0.00012307077122386545, 7.918036135379225e-05, 6.291030149441212e-05, 5.992899968987331e-05, 4.015001104562543e-05, 3.608937549870461e-05, 3.36187076754868e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.05119003914296627, "first_to_last_ratio": 0.7709532409156976, "imbalance_abs_log_ratio": 0.260127554578686, "last_quartile_mean": 0.06639837077818811, "mean": 0.06283537502167746, "normalized": [ 0.7674593499219178, 0.8911005434486201, 0.7949446960721035, 0.7325731497404293, 0.9435544639718605, 0.8683878801940695, 0.8054854824220893, 0.7138473266473999, 0.5978031662245904, 0.5556351000440413, 0.5261832202468875, 0.5057617435862689, 0.715108825077915, 0.6608105011251547, 0.6122576924651686, 0.5785453059609885, 0.7399587078512944, 0.6892519015892086, 0.6970578010795431, 0.8113719810435487, 3.2405178171891893, 2.52495521313829, 1.9895055108588098, 1.5842933381515236, 2.315101350530748, 1.7369258738398763, 1.1174905410660712, 0.8878674720175045, 0.8457916556516879, 0.5666462655907423, 0.5093376096374259, 0.4744685136150315 ], "peak_layer": 21, "peak_normalized": 3.2405178171891893, "population_cv": 0.6532804095990112, "rest_mean": 0.04734907733897368, "spike_contrast": 3.0932256918071666, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.1464613825082779, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 0.048223596066236496, 0.05599263682961464, 0.049950648099184036, 0.04603150859475136, 0.05928859859704971, 0.05456547811627388, 0.050612982362508774, 0.04485486447811127, 0.03756318613886833, 0.03491353988647461, 0.03306291997432709, 0.03177972882986069, 0.04493413120508194, 0.041522275656461716, 0.038471441715955734, 0.036353111267089844, 0.04649558290839195, 0.04330940172076225, 0.04379988834261894, 0.05098286271095276, 0.20361915230751038, 0.158656507730484, 0.12501132488250732, 0.09954966604709625, 0.1454702615737915, 0.10914038866758347, 0.07021793723106384, 0.055789485573768616, 0.0531456358730793, 0.035605430603027344, 0.03200441971421242, 0.029813406988978386 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 2.293479383297381e-06, "first_to_last_ratio": 0.445969741543177, "imbalance_abs_log_ratio": 0.8075041733371805, "last_quartile_mean": 5.142679356140434e-06, "mean": 3.873292421019414e-06, "normalized": [ 0.6548345314656568, 0.6098593868708077, 0.5890922218088536, 0.6012559418227537, 0.5982935563608919, 0.5743111722490878, 0.5597275993993273, 0.5496382066817245, 0.4494881925876058, 0.4317042818847807, 0.42858151997472205, 0.44091762118925537, 0.5606916778798313, 0.5514610505577893, 0.5418155695170769, 0.543856847054833, 0.6362195338062625, 0.6499443412604623, 0.6710047275371834, 0.7226720079645516, 3.0150009122712964, 2.639960940380615, 2.3465929682489137, 2.0112500416813845, 2.8998499180306, 2.237539331592885, 1.506545124394083, 1.1285810413622663, 1.0054446853908898, 0.6799751247000538, 0.607691428375801, 0.5561984956977537 ], "peak_layer": 21, "peak_normalized": 3.0150009122712964, "population_cv": 0.7767874334961873, "rest_mean": 2.7381803546530946e-06, "spike_contrast": 3.653118598414905, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.000289757939754e-05, "top_five_layers": [ 21, 25, 22, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 2.5363656277477276e-06, 2.362163741054246e-06, 2.2817264380137203e-06, 2.3288400825549616e-06, 2.3173658973973943e-06, 2.224475110779167e-06, 2.167988668588805e-06, 2.128909500243026e-06, 1.7409992096872884e-06, 1.6721169231459498e-06, 1.6600215531070717e-06, 1.707802880446252e-06, 2.1717228264606092e-06, 2.135969907612889e-06, 2.0986101390008116e-06, 2.106516603816999e-06, 2.4642642983963015e-06, 2.5174244910886046e-06, 2.5989975256379694e-06, 2.799120011331979e-06, 1.1677980182867032e-05, 1.0225340702163521e-05, 9.089040759135969e-06, 7.790159543219488e-06, 1.1231966709601693e-05, 8.666644134791568e-06, 5.835289812239353e-06, 4.371324394014664e-06, 3.8943812796787824e-06, 2.6337424969824497e-06, 2.353766603846452e-06, 2.1543194179685088e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 4.442123326953151e-05, "first_to_last_ratio": 0.9390502622041328, "imbalance_abs_log_ratio": 0.06288627383246957, "last_quartile_mean": 4.730442560685333e-05, "mean": 5.047756872045284e-05, "normalized": [ 0.8442457511608136, 0.9554189601166778, 0.8651399119276457, 0.8073037852085534, 1.011773735449473, 0.9346086902233326, 0.859731545150837, 0.7619317954676181, 0.657244201472419, 0.6123980431859172, 0.5782989976765822, 0.5555348776283999, 0.7735150062029652, 0.7093353168527977, 0.6548347172091762, 0.6154299343077795, 0.8164721063408712, 0.7613444152108153, 0.7654949233837004, 0.8721362954625393, 3.2652345954629065, 2.4905637834711873, 1.8895133502771215, 1.4453947777525853, 2.0912180007544503, 1.5360217136102785, 1.0169235828420913, 0.8253134404517026, 0.6717824033622655, 0.48834596354787496, 0.4437661443704546, 0.4237292344581673 ], "peak_layer": 21, "peak_normalized": 3.2652345954629065, "population_cv": 0.6182834768826335, "rest_mean": 3.892021615886026e-05, "spike_contrast": 2.900479074737969, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00011288727255305275, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 4.2615472921170294e-05, 4.8227226216113195e-05, 4.367015935713425e-05, 4.075073229614645e-05, 5.107187826070003e-05, 4.717677438748069e-05, 4.3397158151492476e-05, 3.846046456601471e-05, 3.317608934594318e-05, 3.091236430918798e-05, 2.9191127396188676e-05, 2.8042049962095916e-05, 3.904515688191168e-05, 3.580552220228128e-05, 3.305446443846449e-05, 3.106540680164471e-05, 4.1213526856154203e-05, 3.843081503873691e-05, 3.8640322600258514e-05, 4.402331978781149e-05, 0.00016482110368087888, 0.00012571760453283787, 9.537803998682648e-05, 7.296001422218978e-05, 0.00010555960034253076, 7.753464160487056e-05, 5.133183003636077e-05, 4.165981590631418e-05, 3.390994243090972e-05, 2.4650516934343614e-05, 2.2400236048270017e-05, 2.138882155122701e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 3.35970580636058e-05, "first_to_last_ratio": 1.183082469649485, "imbalance_abs_log_ratio": 0.16812329486526428, "last_quartile_mean": 2.8397900337040483e-05, "mean": 3.5426135951865945e-05, "normalized": [ 0.9325726638234952, 1.0264228324198814, 0.9395008785893599, 0.8896754656866502, 1.0823546164520876, 1.002292895319797, 0.9051263930494292, 0.8090082728462467, 0.7240290452324956, 0.6814307833477395, 0.6400266234002182, 0.6142285583444715, 0.8434981841324323, 0.7676858556869983, 0.7064344235164509, 0.6656552980925776, 0.9160791051424548, 0.8560305037244728, 0.8526061894702537, 0.9377555262910077, 3.3385841431029775, 2.450559245049383, 1.764118224393788, 1.2414543227163002, 1.7869665660194411, 1.2750853185739486, 0.921819584871679, 0.7867281257526472, 0.5090598376007598, 0.4052896271109379, 0.3729632040504296, 0.3549576861891871 ], "peak_layer": 21, "peak_normalized": 3.3385841431029775, "population_cv": 0.5922680449883523, "rest_mean": 2.8102526947030264e-05, "spike_contrast": 2.6678606062470647, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 7.497362457797863e-05, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 3.303744597360492e-05, 3.6362194805406034e-05, 3.328288585180417e-05, 3.151776400045492e-05, 3.834364179056138e-05, 3.5507364373188466e-05, 3.206513065379113e-05, 2.8660037060035393e-05, 2.5649551389506087e-05, 2.414045957266353e-05, 2.2673670173389837e-05, 2.175974441342987e-05, 2.98818813462276e-05, 2.7196143491892144e-05, 2.5026241928571835e-05, 2.3581595087307505e-05, 3.24531429214403e-05, 3.032585300388746e-05, 3.0204542781575583e-05, 3.3221054763998836e-05, 0.00011827313574030995, 8.681384497322142e-05, 6.249589205253869e-05, 4.3979929614579305e-05, 6.330532050924376e-05, 4.5171345846029e-05, 3.265650593675673e-05, 2.7870737540069968e-05, 1.8034023014479317e-05, 1.435784542991314e-05, 1.3212645171734039e-05, 1.2574779248097911e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.0001175618963316083, "first_to_last_ratio": 0.7472280359888956, "imbalance_abs_log_ratio": 0.2913848713729993, "last_quartile_mean": 0.00015733068175904918, "mean": 0.0001475783419664367, "normalized": [ 0.7539335055517485, 0.8485908925894718, 0.7671380034067055, 0.7221187547562506, 0.9161421981299216, 0.8510423035463365, 0.8040139845348296, 0.7098739762546249, 0.5921519684486223, 0.5526890935329054, 0.5267158319052243, 0.5064890538358937, 0.699155203785276, 0.6460998215976334, 0.6066243745851786, 0.5643569853524535, 0.729020737260662, 0.6850322033333521, 0.6876316692176827, 0.7819456161539314, 3.243720072824733, 2.5794511063238676, 2.043978587180185, 1.6534243806222502, 2.3371192579790736, 1.7969709048957057, 1.167004731785933, 0.9411888715768515, 0.8017105792030909, 0.5375092481175581, 0.48493150700127435, 0.4622245747107726 ], "peak_layer": 21, "peak_normalized": 3.243720072824733, "population_cv": 0.6705093936976806, "rest_mean": 0.00011009511891407547, "spike_contrast": 3.178957885701889, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0003499877464491874, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 0.00011126425670227036, 0.00012523363693617284, 0.00011321295460220426, 0.00010656908852979541, 0.00013520274660550058, 0.00012559541210066527, 0.00011865505075547844, 0.0001047620244207792, 8.738880569580942e-05, 8.156494004651904e-05, 7.773184916004539e-05, 7.474681478925049e-05, 0.0001031801657518372, 9.535034041618928e-05, 8.952461939770728e-05, 8.328686817549169e-05, 0.00010758767166407779, 0.00010109591676155105, 0.00010147954162675887, 0.00011539823753992096, 0.00047870283015072346, 0.00038067111745476723, 0.0003016469709109515, 0.00024400962865911424, 0.00034490818507038057, 0.0002651939867064357, 0.00017222462338395417, 0.00013889909314457327, 0.00011831511801574379, 7.932472362881526e-05, 7.156538777053356e-05, 6.821433635195717e-05 ] } } }, "post_attention_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "90a633f12d984935028fe5d7212be48e7da7b4a0806751939a551938d16dc540", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.0933690071105957, 0.35791492462158203, 0.3940892517566681, 0.4057151973247528, 0.09380833059549332, 0.11286329478025436, 0.13231655955314636, 0.1621972620487213, 0.05156813934445381, 0.08533531427383423, 0.12003090232610703, 0.16063299775123596, 0.046871740370988846, 0.079384446144104, 0.11459959298372269, 0.15625107288360596, 0.08397205173969269, 0.11654024571180344, 0.15294702351093292, 0.21572710573673248, 0.03708760067820549, 0.06553087383508682, 0.09390194714069366, 0.12852425873279572, 0.025487789884209633, 0.06059965491294861, 0.08860157430171967, 0.12100546807050705, 0.024166326969861984, 0.06877554953098297, 0.12682415544986725, 0.2025238573551178 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.4657116139460413e-05, "first_to_last_ratio": 0.6094207983868245, "imbalance_abs_log_ratio": 0.49524628372702406, "last_quartile_mean": 2.4050895831351227e-05, "mean": 2.012364140568934e-05, "normalized": [ 0.6757810223737426, 0.7862782366272546, 0.6956742911341667, 0.645696070130233, 0.8486116417698539, 0.7866567023571445, 0.7355711504661757, 0.6525555016113147, 0.5676629357160193, 0.5065196141176592, 0.4765477406007062, 0.4567702622869, 0.6387850255773179, 0.6012516562554105, 0.5676978717094758, 0.5320979135959467, 0.660140815357474, 0.6199672249932086, 0.6251969579127784, 0.6703296515344952, 3.1476456930462797, 2.6843935159061667, 2.147025871519746, 1.7098926204474763, 2.310003501268429, 2.02536057377974, 1.5260208260637402, 0.9852869427838804, 1.0403527573808369, 0.7424129105715245, 0.4909974573147262, 0.4408150437901761 ], "peak_layer": 21, "peak_normalized": 3.1476456930462797, "population_cv": 0.7023455047176551, "rest_mean": 1.4907175278090614e-05, "spike_contrast": 3.239551262652627, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.829255849472247e-05, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 1.3599174963019323e-05, 1.582278127898462e-05, 1.3999499969941098e-05, 1.2993756172363646e-05, 1.7077156371669844e-05, 1.5830397387617268e-05, 1.4802370060351677e-05, 1.3131792911735829e-05, 1.1423445357650053e-05, 1.0193019079451915e-05, 9.589875844540074e-06, 9.191880963044241e-06, 1.2854680790042039e-05, 1.2099372725060675e-05, 1.1424148397054523e-05, 1.0707747605920304e-05, 1.3284437045513187e-05, 1.2475998119043652e-05, 1.2581239388964605e-05, 1.3489473531080876e-05, 6.334209319902584e-05, 5.4019772505853325e-05, 4.3205978727201e-05, 3.440926593611948e-05, 4.648568210541271e-05, 4.07576299039647e-05, 3.0709095881320536e-05, 1.982756111829076e-05, 2.0935685824952088e-05, 1.4940051187295467e-05, 9.88065676210681e-06, 8.870803867466748e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 5.829710562466062e-05, "first_to_last_ratio": 0.6072508868460227, "imbalance_abs_log_ratio": 0.49881325067107773, "last_quartile_mean": 9.600168050383218e-05, "mean": 8.04003983603252e-05, "normalized": [ 0.6783620403407459, 0.7863497478511254, 0.7000374993610626, 0.6471153960919661, 0.8353499122055624, 0.7743388664853165, 0.7293878791763712, 0.6497369450201427, 0.570732789612889, 0.5105438596817938, 0.4799740390322282, 0.4590044982477276, 0.6477850704647679, 0.6029712759057101, 0.5684996518209374, 0.5312286531130485, 0.6575848951936205, 0.6204730346458087, 0.629222556567805, 0.6709457586481146, 3.1526166331987144, 2.693185986758889, 2.1505035942092414, 1.701690725446724, 2.3039587527644554, 2.018829142146718, 1.519765537386825, 0.9812068005325093, 1.0399315417453956, 0.7451740498188497, 0.49646309627653795, 0.44702977024839613 ], "peak_layer": 21, "peak_normalized": 3.1526166331987144, "population_cv": 0.7020174312406612, "rest_mean": 5.955002699112002e-05, "spike_contrast": 3.2408449417296112, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00019299240375403315, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 5.454057827591896e-05, 6.322283297777176e-05, 5.628329381579533e-05, 5.202833563089371e-05, 6.71624657115899e-05, 6.22571533313021e-05, 5.8643076044972986e-05, 5.223910920904018e-05, 4.588714364217594e-05, 4.1047929698834196e-05, 3.859010394080542e-05, 3.6904144508298486e-05, 5.2082177717238665e-05, 4.8479130782652646e-05, 4.5707598474109545e-05, 4.27109953307081e-05, 5.287008752929978e-05, 4.988627915736288e-05, 5.058974420535378e-05, 5.3944306273479015e-05, 0.00025347163318656385, 0.00021653322619386017, 0.00017290134564973414, 0.00013681661221198738, 0.0001852392015280202, 0.00016231466725002974, 0.00012218975462019444, 7.88894176366739e-05, 8.361091022379696e-05, 5.9912290453212336e-05, 3.991583071183413e-05, 3.59413716068957e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.05169845558702946, "first_to_last_ratio": 0.6072508754048147, "imbalance_abs_log_ratio": 0.49881326951206806, "last_quartile_mean": 0.08513525081798434, "mean": 0.0712998784147203, "normalized": [ 0.6783619951609938, 0.786349724122647, 0.7000374743682577, 0.6471153772114631, 0.8353498624040391, 0.774338855687226, 0.729387848399732, 0.6497369828883814, 0.5707327933468814, 0.5105438598068986, 0.4799740650385461, 0.45900445901159087, 0.6477850423892585, 0.6029712911228355, 0.5684996530360322, 0.5312286608963362, 0.6575848715023773, 0.6204730273795, 0.6292225631787356, 0.6709457289261637, 3.152616671814094, 2.693186128732524, 2.150503721956032, 1.7016907445638214, 2.303958774916342, 2.0188290625883343, 1.5197655429953278, 0.981206773145755, 1.039931555380646, 0.7451740672617071, 0.49646307443703724, 0.4470297463304832 ], "peak_layer": 21, "peak_normalized": 3.152616671814094, "population_cv": 0.70201745433721, "rest_mean": 0.052809559360698415, "spike_contrast": 3.2408450927884243, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.17114760130643844, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 0.048367127776145935, 0.056066639721393585, 0.04991258680820465, 0.046139247715473175, 0.059560343623161316, 0.05521026626229286, 0.05200526490807533, 0.0463261678814888, 0.04069317877292633, 0.036401715129613876, 0.03422209247946739, 0.0327269621193409, 0.04618699476122856, 0.04299177974462509, 0.04053395614027977, 0.037876538932323456, 0.04688572138547897, 0.044239651411771774, 0.044863492250442505, 0.047838348895311356, 0.22478118538856506, 0.1920238435268402, 0.15333065390586853, 0.12133034318685532, 0.1642719805240631, 0.14394226670265198, 0.10835909843444824, 0.06995992362499237, 0.07414699345827103, 0.05313082039356232, 0.03539775684475899, 0.03187316656112671 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 2.317259230721902e-06, "first_to_last_ratio": 0.35665874650259244, "imbalance_abs_log_ratio": 1.0309758465083512, "last_quartile_mean": 6.497132773120029e-06, "mean": 4.492468697492313e-06, "normalized": [ 0.5694283261976025, 0.527162238879597, 0.5069677223499952, 0.5169219758743683, 0.5167285360874581, 0.5011532928366966, 0.49061467097646116, 0.4975013703289752, 0.41211554520475835, 0.3863198030086292, 0.3787459166580058, 0.38785476948100367, 0.4953856795983567, 0.493696852070983, 0.4965364995551244, 0.49228294995504224, 0.553135351492583, 0.5731144625451857, 0.5949797886628256, 0.6269543934401883, 2.991422102255377, 2.8607693678126824, 2.46602465979906, 2.0943619443179213, 2.767059684919099, 2.477440137836224, 1.9032189385510152, 1.2285251544343567, 1.2121681046978803, 0.874835232215944, 0.589970244793492, 0.5166042831631082 ], "peak_layer": 21, "peak_normalized": 2.991422102255377, "population_cv": 0.8381704476164953, "rest_mean": 3.1314773423218867e-06, "spike_contrast": 3.7815448495732364, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.1841822015412617e-05, "top_five_layers": [ 21, 22, 25, 26, 23 ], "top_five_spike_overlap": 4 }, "values": [ 2.5581389309081715e-06, 2.3682598566665547e-06, 2.277536623296328e-06, 2.3222557956614764e-06, 2.3213867734739324e-06, 2.2514154807140585e-06, 2.2040710518922424e-06, 2.235009333162452e-06, 1.8514161865823553e-06, 1.7355296222376637e-06, 1.7015041748891235e-06, 1.742425411066506e-06, 2.225504658781574e-06, 2.2179176539793843e-06, 2.2306746814138023e-06, 2.211565742982202e-06, 2.484943252056837e-06, 2.5746987830643775e-06, 2.6729280762083363e-06, 2.8165729872853262e-06, 1.3438870155368932e-05, 1.2851916835643351e-05, 1.1078538591391407e-05, 9.408855476067401e-06, 1.2430929018591996e-05, 1.1129822269140277e-05, 8.550151505914982e-06, 5.519110800378257e-06, 5.445627266453812e-06, 3.930169896193547e-06, 2.6504228571866406e-06, 2.3208285711007193e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 4.480992902244907e-05, "first_to_last_ratio": 0.7542989526231301, "imbalance_abs_log_ratio": 0.28196650066503776, "last_quartile_mean": 5.940606024523731e-05, "mean": 5.624609735832564e-05, "normalized": [ 0.7593189081163986, 0.8584899766461837, 0.7758411348304551, 0.7257032038756371, 0.9115949579325926, 0.8478585798985628, 0.791306819195711, 0.703296291763442, 0.6342085768594776, 0.5708926166563143, 0.5360034021261918, 0.5119200900833328, 0.7122730532881045, 0.6576365221836818, 0.6170512605021473, 0.5742091312248548, 0.7380382630332617, 0.6958730711433307, 0.702031804287528, 0.7406728598801368, 3.1985536041966367, 2.658074789581204, 2.066704719937707, 1.5629982359965144, 2.1587687220561924, 1.7912910686863504, 1.3377069279004006, 0.8975311406544447, 0.8407126410078246, 0.5976870739310991, 0.4315898849446666, 0.3941606675796141 ], "peak_layer": 21, "peak_normalized": 3.1985536041966367, "population_cv": 0.6612253265050619, "rest_mean": 4.240309937341846e-05, "spike_contrast": 3.089356401125345, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013099828647682442, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 4.2708725231932476e-05, 4.828671080758795e-05, 4.363803600426763e-05, 4.081797305843793e-05, 5.1273658755235374e-05, 4.7688736231066287e-05, 4.450792039278895e-05, 3.9557671698275954e-05, 3.5671757359523326e-05, 3.2110481697600335e-05, 3.0148099540383555e-05, 2.8793507226509973e-05, 4.00625795009546e-05, 3.698948785313405e-05, 3.4706725273281336e-05, 3.229702269891277e-05, 4.151177199673839e-05, 3.9140144508564845e-05, 3.948654921259731e-05, 4.1659957787487656e-05, 0.0001799061574274674, 0.00014950633340049535, 0.0001162440748885274, 8.791255095275119e-05, 0.00012142231571488082, 0.00010075313184643164, 7.524079410359263e-05, 5.0482623919378966e-05, 4.728680505650118e-05, 3.3617565350141376e-05, 2.427524668746628e-05, 2.2169999283505604e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 3.384801061656617e-05, "first_to_last_ratio": 0.9740047819331958, "imbalance_abs_log_ratio": 0.02633906576931732, "last_quartile_mean": 3.475138032627001e-05, "mean": 3.8690275459885015e-05, "normalized": [ 0.8541538250770527, 0.9415230863648691, 0.8597053468534278, 0.816066837065049, 0.9910309334070734, 0.9267294340872885, 0.8523049476412223, 0.7572490181408758, 0.7081191176405685, 0.6443517927963444, 0.6037734323315225, 0.5756116900407472, 0.7887443909138365, 0.7248706260931408, 0.6750345264379841, 0.633794207108887, 0.8434851920289833, 0.7938278400728305, 0.7971348135467743, 0.8220444008320537, 3.298671908004925, 2.612487605975051, 1.950851070533051, 1.3428804993059804, 1.9255864325842058, 1.4942946568254274, 1.1186305566147283, 0.822664893239064, 0.6568214429021311, 0.4646495190211241, 0.3623372415841501, 0.34056871492963064 ], "peak_layer": 21, "peak_normalized": 3.298671908004925, "population_cv": 0.6252780463659825, "rest_mean": 2.9905465689134196e-05, "spike_contrast": 2.8800169546676924, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 8.612824822193944e-05, "top_five_layers": [ 21, 22, 23, 25, 26 ], "top_five_spike_overlap": 4 }, "values": [ 3.304744677734561e-05, 3.64277875632979e-05, 3.3262236684095114e-05, 3.157385071972385e-05, 3.834325980278663e-05, 3.5855417081620544e-05, 3.297591320006177e-05, 2.9298173103597946e-05, 2.7397323719924316e-05, 2.4930148356361315e-05, 2.336016041226685e-05, 2.227057484560646e-05, 3.0516737751895562e-05, 2.8045444196322933e-05, 2.611727177281864e-05, 2.452167245792225e-05, 3.263467442593537e-05, 3.071341780014336e-05, 3.084136551478878e-05, 3.1805124308448285e-05, 0.00012762652477249503, 0.00010107786511071026, 7.547896530013531e-05, 5.195641642785631e-05, 7.450146949850023e-05, 5.781467189081013e-05, 4.3280124373268336e-05, 3.182913133059628e-05, 2.541260255384259e-05, 1.7977417883230373e-05, 1.401892768626567e-05, 1.3176697393646464e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00011881819409609307, "first_to_last_ratio": 0.5858660432005869, "imbalance_abs_log_ratio": 0.5346641107578359, "last_quartile_mean": 0.0002028077842624043, "mean": 0.0001675532937497337, "normalized": [ 0.6676405577533395, 0.7480364199301687, 0.6769763115579581, 0.6368334430352999, 0.8101874659576359, 0.7560101568775733, 0.7242722329347805, 0.6531372168922783, 0.569150085590359, 0.5080318969181871, 0.47519098467081156, 0.45888126162494614, 0.6302390928396696, 0.5851986956962459, 0.5573344319393583, 0.5133335361358992, 0.6455392341889507, 0.6148024241191441, 0.6209170612517536, 0.6589570869141987, 3.121371094689797, 2.6936365726462816, 2.2146820529210776, 1.7763796870270414, 2.3963701078287363, 2.056340004797266, 1.572482658633456, 1.024072444271845, 1.0322195327669619, 0.7057443743535575, 0.46898398312387723, 0.42704789011154415 ], "peak_layer": 21, "peak_normalized": 3.121371094689797, "population_cv": 0.7183385489551041, "rest_mean": 0.00012285727657231123, "spike_contrast": 3.328348128139876, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0004089117865078151, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 0.00011186537449248135, 0.00012533596600405872, 0.0001134296107920818, 0.0001067035409505479, 0.00013574957847595215, 0.0001266719918930903, 0.00012135419819969684, 0.00010943529196083546, 9.53629714786075e-05, 8.512241765856743e-05, 7.96198146417737e-05, 7.6887066825293e-05, 0.00010559863585513085, 9.805196896195412e-05, 9.338321979157627e-05, 8.601072477176785e-05, 0.0001081622249330394, 0.00010301217116648331, 0.00010403669875813648, 0.00011041043035220355, 0.0005229960079304874, 0.00045132767991162837, 0.0003710772725753486, 0.0002976382675115019, 0.00040151970461010933, 0.0003445465408731252, 0.0002634746488183737, 0.00017158671107608825, 0.00017295178258791566, 0.00011824979446828365, 7.857981108827516e-05, 7.155328057706356e-05 ] } } }, "post_mlp_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "89f2226456bfe96e245d2aad0f699d32c676f9f53099789cd870253d002402f5", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.16915453970432281, 0.374446302652359, 0.39966297149658203, 0.4194087088108063, 0.09892481565475464, 0.11664740741252899, 0.13951337337493896, 0.18593810498714447, 0.06607726216316223, 0.10425819456577301, 0.14551182091236115, 0.19865906238555908, 0.06484340131282806, 0.10430748760700226, 0.14720691740512848, 0.20439592003822327, 0.10052042454481125, 0.14085541665554047, 0.1745898574590683, 0.2420409768819809, 0.049456026405096054, 0.08509506285190582, 0.11783231794834137, 0.15649692714214325, 0.05533855780959129, 0.08507220447063446, 0.11942370235919952, 0.1559232622385025, 0.06628894805908203, 0.12384890764951706, 0.19928614795207977, 0.30097496509552 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.4510050277749542e-05, "first_to_last_ratio": 0.7743096996079971, "imbalance_abs_log_ratio": 0.255783356730045, "last_quartile_mean": 1.873933683782525e-05, "mean": 1.772222358908948e-05, "normalized": [ 0.7655738518273417, 0.8917418894414295, 0.7902270979958571, 0.7316384060873433, 0.9592931634098782, 0.8828162060633833, 0.8112185990320713, 0.7174816289091832, 0.5935539930427093, 0.5514073845754808, 0.522640268952773, 0.50377149417381, 0.7044978089956251, 0.6589104369318574, 0.6114200884798895, 0.5793087411391302, 0.7435486231843338, 0.6887380175928803, 0.692133312159999, 0.8093969639662545, 3.2325450017939272, 2.512318244382783, 1.9971229388240013, 1.5895601132651613, 2.322861628648705, 1.745234574445335, 1.1261863634676146, 0.8913245597041289, 0.8429168761495727, 0.5609154923495704, 0.5024744978075042, 0.4672217332004658 ], "peak_layer": 21, "peak_normalized": 3.2325450017939272, "population_cv": 0.6537061924560114, "rest_mean": 1.3354412287838016e-05, "spike_contrast": 3.0932401760178796, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.1308404615847397e-05, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 1.356767097604461e-05, 1.5803649148438126e-05, 1.4004581316839904e-05, 1.2966259419044945e-05, 1.7000807929434814e-05, 1.564546619192697e-05, 1.4376597391674295e-05, 1.2715369848592672e-05, 1.0519096576899756e-05, 9.77216495812172e-06, 9.262347703042906e-06, 8.927951057557948e-06, 1.2485267689044122e-05, 1.1677358088491019e-05, 1.0835723514901474e-05, 1.0266639037581626e-05, 1.3177334949432407e-05, 1.220596914208727e-05, 1.2266141311556567e-05, 1.4344313967740163e-05, 5.728788528358564e-05, 4.452386565390043e-05, 3.5393459256738424e-05, 2.817053973558359e-05, 4.116627314942889e-05, 3.092943734372966e-05, 1.9958526536356658e-05, 1.579625313752331e-05, 1.4938361346139573e-05, 9.940669770003296e-06, 8.90496539796004e-06, 8.280208021460567e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 5.772373378931661e-05, "first_to_last_ratio": 0.7709541968266947, "imbalance_abs_log_ratio": 0.26012631467157654, "last_quartile_mean": 7.487310404030723e-05, "mean": 7.085559036568156e-05, "normalized": [ 0.767456541889667, 0.8910969874796977, 0.7949526654342541, 0.7325741840266811, 0.943544133913626, 0.868390294625501, 0.8054913948148298, 0.7138323441642012, 0.5978064063949543, 0.5556378314836203, 0.5261854153191788, 0.505768953649729, 0.715108334510866, 0.6608161569684796, 0.612256437426044, 0.5785464585899268, 0.739960161627241, 0.6892568994732284, 0.697060043562065, 0.8113764458982461, 3.2405349047340315, 2.5249733605503017, 1.9894958890200054, 1.5842775624032772, 2.3150907133984013, 1.7369124395330813, 1.1174777605817472, 0.8878712764898554, 0.8457898829566253, 0.5666580094993431, 0.5093321459106411, 0.474467963670652 ], "peak_layer": 21, "peak_normalized": 3.2405349047340315, "population_cv": 0.6532808754860918, "rest_mean": 5.3392646400930564e-05, "spike_contrast": 3.093225357948515, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00016515548777533694, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 5.437858635559678e-05, 6.313920312095433e-05, 5.632684042211622e-05, 5.1906976295867935e-05, 6.685537664452568e-05, 6.153030699351802e-05, 5.707356831408106e-05, 5.057901216787286e-05, 4.2357925849501044e-05, 3.9370046579279006e-05, 3.728317824425176e-05, 3.583655779948458e-05, 5.0669423217186704e-05, 4.682251892518252e-05, 4.338179132901132e-05, 4.099325087736361e-05, 5.2430314099183306e-05, 4.8837704525794834e-05, 4.9390600906917825e-05, 5.749055708292872e-05, 0.00022961001377552748, 0.00017890847811941057, 0.00014096690574660897, 0.00011225492198718712, 0.0001640371192479506, 0.00012306995631661266, 7.917954644653946e-05, 6.291064346441999e-05, 5.9928941482212394e-05, 4.015088779851794e-05, 3.608902989071794e-05, 3.36187076754868e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.051189984660595655, "first_to_last_ratio": 0.7709541992191667, "imbalance_abs_log_ratio": 0.26012631156831567, "last_quartile_mean": 0.06639821757562459, "mean": 0.06283544522011653, "normalized": [ 0.7674565953661515, 0.8910969393261454, 0.7949526416572095, 0.7325742284914066, 0.9435441611668938, 0.8683903486633082, 0.8054914004920491, 0.7138323596913118, 0.5978064112769625, 0.5556377993412199, 0.5261854188690636, 0.5057690043708923, 0.7151083226055974, 0.6608161658178341, 0.6122564748857159, 0.5785464975021892, 0.7399601340699639, 0.6892568823568349, 0.6970600459500164, 0.8113764696625058, 3.2405348286343636, 2.524973261140758, 1.9894956995598303, 1.584277576610783, 2.3150909383371494, 1.736912431812052, 1.1174776724854902, 0.8878712823128355, 0.8457898807407633, 0.5666580234128831, 0.5093321198410424, 0.47446798354877795 ], "peak_layer": 21, "peak_normalized": 3.2405348286343636, "population_cv": 0.6532808615381831, "rest_mean": 0.04734913239048587, "spike_contrast": 3.0932253054239998, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.14646153450012206, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 0.048223476856946945, 0.05599247291684151, 0.04995120316743851, 0.04603162780404091, 0.05928801745176315, 0.05456569418311119, 0.0506134107708931, 0.044853974133729935, 0.03756343200802803, 0.03491374850273132, 0.033063095062971115, 0.031780220568180084, 0.04493414983153343, 0.04152267798781395, 0.03847140818834305, 0.036353226751089096, 0.046495724469423294, 0.043309763073921204, 0.043800078332424164, 0.05098320171236992, 0.20362044870853424, 0.15865781903266907, 0.12501084804534912, 0.09954878687858582, 0.1454697698354721, 0.10913966596126556, 0.07021720707416534, 0.05578978732228279, 0.05314558371901512, 0.03560620918869972, 0.032004110515117645, 0.029813406988978386 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 2.293416855536634e-06, "first_to_last_ratio": 0.44595751640176395, "imbalance_abs_log_ratio": 0.8075315862036556, "last_quartile_mean": 5.1426801235265884e-06, "mean": 3.873313932700739e-06, "normalized": [ 0.6547872198845468, 0.6098622810029859, 0.5891095547173398, 0.6012100431571663, 0.5982908205734621, 0.5743244780654446, 0.5596766481388143, 0.5495961168488454, 0.44946743969713066, 0.4316777868522346, 0.4285841881361952, 0.44095385743945303, 0.560667078741325, 0.5514542308757108, 0.5417913687261439, 0.5438534743570708, 0.636286678312979, 0.6499450755816364, 0.6710355180394164, 0.7226628285450687, 3.0150865448734856, 2.639875835373709, 2.3467426593605225, 2.0112905298698336, 2.8997859114535975, 2.237494500860103, 1.506591468158109, 1.1286195048247083, 1.0054186728222039, 0.6799993493915261, 0.607662928646794, 0.5561954066724383 ], "peak_layer": 21, "peak_normalized": 3.0150865448734856, "population_cv": 0.7767952241449113, "rest_mean": 2.738177386163443e-06, "spike_contrast": 3.6531786927134653, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.000305128400214e-05, "top_five_layers": [ 21, 25, 22, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 2.536196461733198e-06, 2.3621880700375186e-06, 2.281806246173801e-06, 2.328675236640265e-06, 2.3173681711341487e-06, 2.2245390027819667e-06, 2.1678033590433188e-06, 2.128758296748856e-06, 1.7409284964742255e-06, 1.6720235862521804e-06, 1.6600411072431598e-06, 1.707952719698369e-06, 2.171639607695397e-06, 2.1359553556976607e-06, 2.0985280571039766e-06, 2.1065152395749465e-06, 2.464538056301535e-06, 2.5174413167405874e-06, 2.59913122135913e-06, 2.79910000244854e-06, 1.1678376722557005e-05, 1.022506785375299e-05, 9.089671038964298e-06, 7.790359632053878e-06, 1.1231781172682531e-05, 8.666518624522723e-06, 5.835501724504866e-06, 4.371497652755352e-06, 3.894302153639728e-06, 2.633850954225636e-06, 2.3536692879133625e-06, 2.1543194179685088e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 4.4421208258427214e-05, "first_to_last_ratio": 0.9390471381412793, "imbalance_abs_log_ratio": 0.06288960067042051, "last_quartile_mean": 4.730455634671671e-05, "mean": 5.047763789889359e-05, "normalized": [ 0.8442417833686091, 0.9554185876612655, 0.8651397352686891, 0.8073036157415787, 1.0117688173530386, 0.9346106525630644, 0.8597336821790709, 0.7619236882899906, 0.6572438052320593, 0.6123997263969396, 0.5783007276195616, 0.5555384405465414, 0.7735135857629635, 0.7093370113557148, 0.654834684626451, 0.6154293791596152, 0.8164712035979397, 0.7613467591474443, 0.7654961084945793, 0.8721378389215614, 3.265225507983762, 2.4905834329623366, 1.889506148192275, 1.4453841483405094, 2.0912250805998966, 1.5360203292388628, 1.0169178649058275, 0.8253190119907806, 0.6717866718190295, 0.48834958251132726, 0.44376373442170963, 0.42372865374700563 ], "peak_layer": 21, "peak_normalized": 3.265225507983762, "population_cv": 0.6182834965857263, "rest_mean": 3.8920269852732655e-05, "spike_contrast": 2.9004789991258666, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00011288742534816265, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 4.261533103999682e-05, 4.822727350983769e-05, 4.367021028883755e-05, 4.0750779589870945e-05, 5.1071699999738485e-05, 4.717693809652701e-05, 4.33973254985176e-05, 3.846010804409161e-05, 3.317611481179483e-05, 3.091249163844623e-05, 2.9191254725446925e-05, 2.8042268240824342e-05, 3.9045138692017645e-05, 3.5805656807497144e-05, 3.305450809421018e-05, 3.106542135355994e-05, 4.1213537770090625e-05, 3.843098602374084e-05, 3.8640435377601534e-05, 4.402345803100616e-05, 0.00016482087085023522, 0.00012571876868605614, 9.537780715618283e-05, 7.295957766473293e-05, 0.00010556010238360614, 7.753467798465863e-05, 5.1331611757632345e-05, 4.166015423834324e-05, 3.3910204365383834e-05, 2.4650733394082636e-05, 2.240014509879984e-05, 2.138882155122701e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 3.359962329341215e-05, "first_to_last_ratio": 1.183161693959634, "imbalance_abs_log_ratio": 0.1681902569407737, "last_quartile_mean": 2.839816693267494e-05, "mean": 3.542719008464701e-05, "normalized": [ 0.9327605618775979, 1.0266348424598963, 0.9395266301163642, 0.8894102418069995, 1.082225986222851, 1.0025232858328599, 0.9050487327854893, 0.8091772559917478, 0.7238157816582237, 0.6814271430375796, 0.6400264742225962, 0.6141081065852076, 0.843581628077757, 0.7675711067089205, 0.7064606405143428, 0.6655561644133133, 0.9162025945827288, 0.8560421033047051, 0.8525829253737746, 0.9377293691727334, 3.3385973508618734, 2.4504448425731913, 1.7642485194416198, 1.2415583752024149, 1.7870101279275883, 1.2751219306613844, 0.9216663623251447, 0.7867121103297002, 0.5090691818722745, 0.4053411321779632, 0.37287136742526056, 0.3549471244558967 ], "peak_layer": 21, "peak_normalized": 3.3385973508618734, "population_cv": 0.592278758078665, "rest_mean": 2.8103131289417097e-05, "spike_contrast": 2.6679271717711734, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 7.497710757888853e-05, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 3.304508572909981e-05, 3.6370787711348385e-05, 3.3284788514720276e-05, 3.150930569972843e-05, 3.834022572846152e-05, 3.5516583011485636e-05, 3.2063333492260426e-05, 2.8666876460192725e-05, 2.5642759283073246e-05, 2.414104892523028e-05, 2.2674339561490342e-05, 2.1756124624516815e-05, 2.9885726689826697e-05, 2.7192887500859797e-05, 2.5027915398823097e-05, 2.3578784748679027e-05, 3.2458483474329114e-05, 3.032716631423682e-05, 3.0204617360141128e-05, 3.322111660963856e-05, 0.00011827712296508253, 8.681237522978336e-05, 6.250236765481532e-05, 4.398492455948144e-05, 6.330874748528004e-05, 4.517398701864295e-05, 3.265204941271804e-05, 2.7870999474544078e-05, 1.8034890672424808e-05, 1.436009733879473e-05, 1.3209784810896963e-05, 1.2574779248097911e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00011756234653148567, "first_to_last_ratio": 0.7472422235887175, "imbalance_abs_log_ratio": 0.29136588457853974, "last_quartile_mean": 0.00015732829706394114, "mean": 0.0001475833446420438, "normalized": [ 0.7536685945005097, 0.8489437148344299, 0.7668873856587688, 0.7219862591209203, 0.9160304874966925, 0.8511301995015967, 0.8040925298742102, 0.7099228291391, 0.592249280993652, 0.5528729353184129, 0.5267845496399454, 0.5066519805725661, 0.6991026141257565, 0.6459439706463571, 0.6067103504076782, 0.5642338800686106, 0.7290006596953322, 0.6851008790441766, 0.687775193830433, 0.7817329016891325, 3.243635755848753, 2.5800278485304857, 2.0438475774053395, 1.653426311601236, 2.3369623381128184, 1.7968342666222143, 1.1670758041878126, 0.9412043950671267, 0.8016088114740478, 0.5374717514485083, 0.4848750369936043, 0.46220890654977165 ], "peak_layer": 21, "peak_normalized": 3.243635755848753, "population_cv": 0.6705219269582756, "rest_mean": 0.0001100977226289817, "spike_contrast": 3.179045807260368, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0003500057035125792, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 0.0001112289319280535, 0.00012528995284810662, 0.00011317980533931404, 0.00010655314690666273, 0.00013519084313884377, 0.00012561264156829566, 0.00011867066496051848, 0.00010477278556209058, 8.74061297508888e-05, 8.159483695635572e-05, 7.774462574161589e-05, 7.47733938624151e-05, 0.00010317590204067528, 9.533057163935155e-05, 8.954034274211153e-05, 8.327152318088338e-05, 0.00010758835560409352, 0.00010110947914654389, 0.00010150416346732527, 0.00011537075624801219, 0.00047870661364868283, 0.0003807691391557455, 0.00030163786141201854, 0.00024401818518526852, 0.0003448967181611806, 0.0002651828108355403, 0.0001722409506328404, 0.00013890609261579812, 0.00011830410949187353, 7.932187872938812e-05, 7.155947969295084e-05, 6.821433635195717e-05 ] } } }, "pre_attention_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "ee695e78937de01b7128d3866b8fc5dd328ebdf2c159be37918d4f02952d5248", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.0626930519938469, 0.09030644595623016, 0.20229491591453552, 0.20300470292568207, 0.19049881398677826, 0.12695962190628052, 0.12132706493139267, 0.11585728824138641, 0.13427142798900604, 0.10485932976007462, 0.10419609397649765, 0.10852954536676407, 0.1286265105009079, 0.09096142649650574, 0.10080574452877045, 0.10566592961549759, 0.1287592649459839, 0.09650235623121262, 0.1065765842795372, 0.11446042358875275, 0.12395420670509338, 0.07221997529268265, 0.07677769660949707, 0.08416415005922318, 0.09094838052988052, 0.059439923614263535, 0.062182098627090454, 0.06896103918552399, 0.0776243805885315, 0.06285123527050018, 0.07231374084949493, 0.08016443997621536 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 9.220272659149487e-06, "first_to_last_ratio": 1.069429605014154, "imbalance_abs_log_ratio": 0.06712542691043744, "last_quartile_mean": 8.621673288189413e-06, "mean": 1.2944014976312701e-05, "normalized": [ 0.6303729714265094, 0.8682643012395536, 0.6280898219637274, 0.5502831812812704, 0.33676078665107495, 0.7338236582315528, 0.9360409006501473, 1.0149193793174633, 0.6122272943195698, 0.5131421981111545, 0.43080038533307174, 0.37792854873844645, 0.39300605492101337, 0.6863011395079349, 0.6243318369244146, 0.5414566873457767, 0.793386920098201, 1.135440497453951, 1.3017177042809938, 1.597588130028887, 2.6171146314958285, 4.210609056409907, 2.860451118900051, 2.2773498949645425, 1.2385768946194602, 1.256602631548737, 0.8025963162001695, 0.5822644883727114, 0.47699344412607797, 0.2914573895713198, 0.32576029794875644, 0.3543414380177253 ], "peak_layer": 22, "peak_normalized": 4.210609056409907, "population_cv": 0.8557638034672043, "rest_mean": 9.010903349243799e-06, "spike_contrast": 3.7934951067201736, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.4182817762484775e-05, "top_five_layers": [ 22, 23, 21, 24, 20 ], "top_five_spike_overlap": 4 }, "values": [ 8.159557182807475e-06, 1.1238826118642464e-05, 8.130004061968066e-06, 7.122873739717761e-06, 4.359036665846361e-06, 9.498624422121793e-06, 1.2116127436456736e-05, 1.3137131645635236e-05, 7.924679266579915e-06, 6.642120297328802e-06, 5.5762866395525634e-06, 4.891912794846576e-06, 5.087076260679169e-06, 8.883492228051182e-06, 8.081360647338443e-06, 7.008623470028397e-06, 1.0269612175761722e-05, 1.4697158803755883e-05, 1.684945345914457e-05, 2.0679204681073315e-05, 3.38759709848091e-05, 5.450218668556772e-05, 3.7025722122052684e-05, 2.9478051146725193e-05, 1.603215787326917e-05, 1.6265483282040805e-05, 1.0388818736828398e-05, 7.536840257671429e-06, 6.174210284370929e-06, 3.7726288155681686e-06, 4.216646175336791e-06, 4.586600880429614e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 3.628570084401872e-05, "first_to_last_ratio": 1.065353016422061, "imbalance_abs_log_ratio": 0.0633062150559101, "last_quartile_mean": 3.405979077797383e-05, "mean": 5.163632124549622e-05, "normalized": [ 0.5687040132298282, 0.8764593520729407, 0.6051388108071001, 0.5571670858837222, 0.3412536542304683, 0.7524665950392966, 0.8942377527803768, 1.0263056508948738, 0.5946205407803693, 0.5239558700278845, 0.42939602856142756, 0.3883393864768877, 0.40120729316033693, 0.7381389545660257, 0.6270734263766993, 0.5557805536677036, 0.7656375942833815, 1.1752771944325544, 1.2670292024807646, 1.589140645161745, 2.652708081192408, 4.256645166896803, 2.872494573593182, 2.263949247584013, 1.2185560925718533, 1.2799157794404965, 0.7966972545457802, 0.5211011847842488, 0.47241308488259476, 0.29979401599371164, 0.32815719518376885, 0.36023871841675353 ], "peak_layer": 22, "peak_normalized": 4.256645166896803, "population_cv": 0.8639609553478637, "rest_mean": 3.5831106625092046e-05, "spike_contrast": 3.823060270757878, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013698448019567878, "top_five_layers": [ 22, 23, 21, 24, 20 ], "top_five_spike_overlap": 4 }, "values": [ 2.9365783120738342e-05, 4.5257136662257835e-05, 3.124714203295298e-05, 2.8770058634108864e-05, 1.762108331604395e-05, 3.885460682795383e-05, 4.6175147872418165e-05, 5.2994648285675794e-05, 3.0704017262905836e-05, 2.7055153623223305e-05, 2.2172431272338144e-05, 2.0052417312399484e-05, 2.0716868675663136e-05, 3.8114780181786045e-05, 3.2379764888901263e-05, 2.86984632111853e-05, 3.953470877604559e-05, 6.06869907642249e-05, 6.542472692672163e-05, 8.205737685784698e-05, 0.00013697608665097505, 0.0002197974972659722, 0.00014832505257800221, 0.00011690201063174754, 6.292175385169685e-05, 6.609014235436916e-05, 4.113851537113078e-05, 2.690774817892816e-05, 2.4393673811573535e-05, 1.5480260117328726e-05, 1.6944830349530093e-05, 1.860140218923334e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.032178521854802966, "first_to_last_ratio": 1.0653530079886349, "imbalance_abs_log_ratio": 0.06330620713982399, "last_quartile_mean": 0.030204562819562852, "mean": 0.04579160569119267, "normalized": [ 0.5687040352411169, 0.8764592932397164, 0.6051388058773625, 0.5571671042879097, 0.3412536282040628, 0.7524665522800605, 0.8942377987777075, 1.0263056510911557, 0.5946205787763836, 0.523955869943047, 0.42939603938950854, 0.388339353567458, 0.4012073030262001, 0.7381389650225099, 0.627073439274379, 0.5557805622734189, 0.7656375421356683, 1.1752771374183286, 1.2670292275109216, 1.589140749744225, 2.6527080853277174, 4.256645179045396, 2.872494621066323, 2.2639491530094316, 1.2185561059252055, 1.2799157254825457, 0.796697271015912, 0.52110118872801, 0.47241309150505434, 0.299794020683871, 0.32815719848905306, 0.3602387226403389 ], "peak_layer": 22, "peak_normalized": 4.256645179045396, "population_cv": 0.8639609572615631, "rest_mean": 0.031775383451194676, "spike_contrast": 3.823060262160724, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.12147920578718185, "top_five_layers": [ 22, 23, 21, 24, 20 ], "top_five_spike_overlap": 4 }, "values": [ 0.026041870936751366, 0.040134478360414505, 0.02771027758717537, 0.025513576343655586, 0.01562655158340931, 0.03445665165781975, 0.04094858467578888, 0.046996183693408966, 0.02722863107919693, 0.023992780596017838, 0.019662734121084213, 0.017782682552933693, 0.018371926620602608, 0.03380056843161583, 0.028714699670672417, 0.02545008435845375, 0.03505977243185043, 0.05381782725453377, 0.058019302785396576, 0.07276930660009384, 0.12147176265716553, 0.19491861760616302, 0.13153614103794098, 0.10366986691951752, 0.05579964071512222, 0.05860939621925354, 0.03648204728960991, 0.02386206015944481, 0.02163255400955677, 0.013728049583733082, 0.015026845037937164, 0.01649590954184532 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 2.4475698694459425e-06, "first_to_last_ratio": 0.4032796404419227, "imbalance_abs_log_ratio": 0.9081250607923619, "last_quartile_mean": 6.069162992616839e-06, "mean": 4.194480464292383e-06, "normalized": [ 1.9615731362727682, 0.7747288452238285, 0.1785692992504359, 0.2633308680375469, 0.18148950204723951, 0.349408887478932, 0.383871487008665, 0.575200617814069, 0.3462103003815746, 0.3364194451166286, 0.2854708899046967, 0.31108734109147806, 0.30067908576983693, 0.513261777831553, 0.5156365685672093, 0.5885731468966555, 0.3799425036372034, 0.7326093628890369, 0.5694277551741158, 0.8619428981160375, 1.5998405056629517, 3.0497744670188736, 2.8846131894263896, 2.4808153927254097, 1.8266821603141778, 2.5325708585606295, 2.0509024328203003, 1.5887038326529273, 1.3182729046192574, 0.967953515357782, 0.6305408473131235, 0.6598961750186662 ], "peak_layer": 22, "peak_normalized": 3.0497744670188736, "population_cv": 0.8409982900394547, "rest_mean": 3.131610654876298e-06, "spike_contrast": 3.1721623566680712, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.93397743513924e-06, "top_five_layers": [ 22, 23, 26, 24, 27 ], "top_five_spike_overlap": 3 }, "values": [ 8.227780199376866e-06, 3.2495850064151455e-06, 7.490054372283339e-07, 1.1045361816286459e-06, 7.612541708112985e-07, 1.4655887525805156e-06, 1.6101414530567126e-06, 2.4126677544700215e-06, 1.4521723414873122e-06, 1.411104790349782e-06, 1.197402070829412e-06, 1.3048497748968657e-06, 1.2611925512828748e-06, 2.1528665001824265e-06, 2.162827513529919e-06, 2.4687585664651124e-06, 1.5936614090605872e-06, 3.072915660595754e-06, 2.3884535949036945e-06, 3.615402647483279e-06, 6.710499746986898e-06, 1.279221942240838e-05, 1.2099453670089133e-05, 1.0405731700302567e-05, 7.661982635909226e-06, 1.0622818990668748e-05, 8.60247018863447e-06, 6.663787189609138e-06, 5.529469945031451e-06, 4.060062110511353e-06, 2.6447912659932626e-06, 2.7679216145770624e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 2.4055149197010905e-05, "first_to_last_ratio": 1.0862063963884478, "imbalance_abs_log_ratio": 0.08269125537722917, "last_quartile_mean": 2.2146020569380198e-05, "mean": 3.472467025744663e-05, "normalized": [ 0.4666666997881285, 0.8663177887052829, 0.6219999695375312, 0.5679169854003736, 0.3844149632749681, 0.7928386669686874, 0.9321622159542684, 0.9095983613860097, 0.6255219017157577, 0.5547521439337422, 0.4549635591797121, 0.4053293755903786, 0.4456534403712468, 0.7958744306526087, 0.6527431991239231, 0.5563097060509357, 0.7807692106312896, 1.1853454205486673, 1.2646459398653336, 1.548550635017449, 2.506691252154866, 4.290967865992318, 3.0302171410000085, 2.2576657022736453, 1.197190310100688, 1.343054897090299, 0.7711430639674054, 0.472191241764576, 0.47986836664550414, 0.29867429600060663, 0.25046027339059757, 0.2895009759231917 ], "peak_layer": 22, "peak_normalized": 4.290967865992318, "population_cv": 0.8713516434982468, "rest_mean": 2.4072257407102302e-05, "spike_contrast": 3.832116701364666, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.224769964930602e-05, "top_five_layers": [ 22, 23, 21, 24, 20 ], "top_five_spike_overlap": 4 }, "values": [ 1.6204847270273603e-05, 3.0082599550951272e-05, 2.1598743842332624e-05, 1.9720730051631108e-05, 1.3348682841751724e-05, 2.7531061277841218e-05, 3.236902557546273e-05, 3.1585503165842965e-05, 2.1721041775890626e-05, 1.926358527271077e-05, 1.579845957166981e-05, 1.4074928913032636e-05, 1.54751687659882e-05, 2.763647717074491e-05, 2.2666292352369055e-05, 1.9317671103635803e-05, 2.7111953386338428e-05, 4.116072886972688e-05, 4.391441325424239e-05, 5.37729101779405e-05, 8.704402716830373e-05, 0.0001490024442318827, 0.00010522329102968797, 7.839669706299901e-05, 4.157203875365667e-05, 4.663713843910955e-05, 2.6777688617585227e-05, 1.6396685168729164e-05, 1.6663270798744634e-05, 1.0371366442996077e-05, 8.697150406078435e-06, 1.0052825928141829e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 1.579084118930041e-05, "first_to_last_ratio": 1.101662799461071, "imbalance_abs_log_ratio": 0.09682067430659903, "last_quartile_mean": 1.433364292324768e-05, "mean": 2.2897426177337366e-05, "normalized": [ 0.39886794777854795, 0.877049729540605, 0.6305457825207785, 0.5772129763101231, 0.4367770087210306, 0.8436499656131583, 0.972408138490726, 0.7805596072336106, 0.6568359938107564, 0.6066017752376727, 0.49315422395705855, 0.4249778682175203, 0.4986390140344086, 0.857095556009823, 0.6846582999905741, 0.5629018786115396, 0.7936422327048538, 1.202661838363613, 1.2736141330113293, 1.5255025761233132, 2.344832382358406, 4.19579128393446, 3.181636935928761, 2.172433813999188, 1.1390303293883177, 1.4232014401744595, 0.759652938746426, 0.4324960642291122, 0.4879317473364259, 0.30796555094004274, 0.21544260190277736, 0.24222836478058168 ], "peak_layer": 22, "peak_normalized": 4.19579128393446, "population_cv": 0.858855507029887, "rest_mean": 1.6084403240610198e-05, "spike_contrast": 3.7109085828539374, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.968775003566407e-05, "top_five_layers": [ 22, 23, 21, 24, 20 ], "top_five_spike_overlap": 4 }, "values": [ 9.133049388765357e-06, 2.0082181436009705e-05, 1.4437875506700948e-05, 1.3216691513662226e-05, 1.0001069313148037e-05, 1.93174128071405e-05, 2.226564356533345e-05, 1.7872805983643048e-05, 1.5039853678899817e-05, 1.3889619367546402e-05, 1.1291962437098846e-05, 9.73089936451288e-06, 1.141755001299316e-05, 1.9625282220658846e-05, 1.5676912880735472e-05, 1.2889004210592248e-05, 1.8172364434576593e-05, 2.7537860660231672e-05, 2.9162485589040443e-05, 3.493008262012154e-05, 5.36906263732817e-05, 9.607282117940485e-05, 7.285129686351866e-05, 4.9743142881197855e-05, 2.6080862880917266e-05, 3.2587649911874905e-05, 1.7394097085343674e-05, 9.903046702675056e-06, 1.117238116421504e-05, 7.051618467812659e-06, 4.933081072522327e-06, 5.546406100620516e-06 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 7.034897362245829e-05, "first_to_last_ratio": 1.0677221298454265, "imbalance_abs_log_ratio": 0.06552752863846958, "last_quartile_mean": 6.588696783182968e-05, "mean": 0.00010326150248829435, "normalized": [ 0.5211538788724873, 0.8562183017948943, 0.6056449715109208, 0.5367927676701035, 0.33719464941795635, 0.7400076028920698, 0.8752614309891937, 0.977887157814224, 0.5891182329173178, 0.4959165872409433, 0.4220040517626212, 0.3715438426401133, 0.3950700830815582, 0.7426266558595574, 0.622546917450552, 0.5356415681171627, 0.7637846157174611, 1.2023092054640232, 1.2927045115765263, 1.5352424858917262, 2.711590866084263, 4.42744668313384, 2.973944015958163, 2.3638740639156786, 1.2453125473146385, 1.2805717504661769, 0.7715376325621908, 0.48453681266643056, 0.47249674362438154, 0.27056316221994425, 0.2589436600974457, 0.3205125432754346 ], "peak_layer": 22, "peak_normalized": 4.42744668313384, "population_cv": 0.9059813522703718, "rest_mean": 6.990356949380064e-05, "spike_contrast": 4.054075388577877, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0002833943406585604, "top_five_layers": [ 22, 23, 21, 24, 20 ], "top_five_spike_overlap": 4 }, "values": [ 5.381513255997561e-05, 8.841438830131665e-05, 6.25398097326979e-05, 5.5430027714464813e-05, 3.481922612991184e-05, 7.641429692739621e-05, 9.03808104339987e-05, 0.00010097809717990458, 6.083323387429118e-05, 5.120909190736711e-05, 4.357677244115621e-05, 3.8366175431292504e-05, 4.079553036717698e-05, 7.66847442719154e-05, 6.428513006540015e-05, 5.5311153118964285e-05, 7.886954699642956e-05, 0.00012415225501172245, 0.00013348661013878882, 0.0001585314457770437, 0.0002800029469653964, 0.00045718479668721557, 0.00030709392740391195, 0.00024409718753304332, 0.00012859284470323473, 0.00013223376299720258, 7.96701351646334e-05, 5.003399928682484e-05, 4.879072366748005e-05, 2.7938758648815565e-05, 2.6738911401480436e-05, 3.309660678496584e-05 ] } } }, "pre_mlp_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "e53fa425d89dfeb0e9395eca9bcd73dc0a55314abf45c5b10b43e2cbfa724345", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.059514161199331284, 0.08157797157764435, 0.14292451739311218, 0.10997853428125381, 0.0991024449467659, 0.1065012738108635, 0.12155740708112717, 0.1095810979604721, 0.06964563578367233, 0.08311934769153595, 0.09363401681184769, 0.0941167026758194, 0.08564268052577972, 0.09493856877088547, 0.09063401818275452, 0.09473779052495956, 0.10505300015211105, 0.09355910867452621, 0.09618847072124481, 0.11416846513748169, 0.047205954790115356, 0.053352247923612595, 0.06512638926506042, 0.07829347252845764, 0.05368548259139061, 0.05310077592730522, 0.061235878616571426, 0.07320034503936768, 0.05304349213838577, 0.0639304369688034, 0.07404901832342148, 0.08157160878181458 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 4.406091136388568e-06, "first_to_last_ratio": 0.13952873923075965, "imbalance_abs_log_ratio": 1.9694846829479884, "last_quartile_mean": 3.15783770474809e-05, "mean": 1.6862885999557875e-05, "normalized": [ 0.5744948983086973, 0.20976707009476936, 0.1380798037056023, 0.25679297795080097, 0.1911977573081938, 0.15397727234373496, 0.20783011320461262, 0.35817424975363493, 0.40845309421862647, 0.30185320868879156, 0.2734882217077138, 0.2725610842103788, 0.35283552344917235, 0.37101154440150896, 0.4205580365391262, 0.41850822128869175, 0.3521537888379847, 0.4454210973675298, 0.41314047811549354, 0.5286573606706721, 2.6023136902657806, 2.9811111131589443, 2.583962732113176, 2.2024122663803394, 2.8672118131663793, 2.8994466418579012, 2.457231935453014, 1.624795250078144, 1.8440638614795903, 1.4598750180798175, 0.9499358030010668, 0.8786840728001112 ], "peak_layer": 22, "peak_normalized": 2.9811111131589443, "population_cv": 0.9711949879643771, "rest_mean": 1.1718449412809503e-05, "spike_contrast": 3.8096203683057026, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.464284356799908e-05, "top_five_layers": [ 22, 26, 25, 21, 23 ], "top_five_spike_overlap": 4 }, "values": [ 9.687641977507155e-06, 3.5372781894693617e-06, 2.3284239887289004e-06, 4.330270712671336e-06, 3.2241459848592058e-06, 2.596501190055278e-06, 3.5046155062445905e-06, 6.039851541572716e-06, 6.88769796397537e-06, 5.090116246719845e-06, 4.611800704878988e-06, 4.596166490955511e-06, 5.949825208517723e-06, 6.2563253777625505e-06, 7.091822226357181e-06, 7.05725642546895e-06, 5.938329195487313e-06, 7.511085186706623e-06, 6.9667407842644025e-06, 8.914688805816695e-06, 4.388251909404062e-05, 5.027013685321435e-05, 4.357306897873059e-05, 3.7139026971999556e-05, 4.8349465942010283e-05, 4.88930381834507e-05, 4.1436022002017125e-05, 2.739873707469087e-05, 3.1096238672034815e-05, 2.4617706003482454e-05, 1.6018659152905457e-05, 1.4817149349255487e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 1.77339601350468e-05, "first_to_last_ratio": 0.14031202708197554, "imbalance_abs_log_ratio": 1.9638865715137788, "last_quartile_mean": 0.00012638945145226899, "mean": 6.748338435613732e-05, "normalized": [ 0.581715983352747, 0.21329408694826651, 0.1383654282103214, 0.26055913189981234, 0.18941324497814624, 0.15268445345034073, 0.20588194209370914, 0.3604059140126238, 0.4112482421872158, 0.3050474729826253, 0.2735060049358427, 0.27320166025083037, 0.35640352937895586, 0.3762475521803712, 0.424911703962158, 0.41981156521974744, 0.35303414718819204, 0.4487530978367584, 0.4155766160707765, 0.5338541629759799, 2.603208152672671, 2.9606390037219725, 2.573692405652612, 2.185365730719144, 2.873997131282303, 2.8916679373421434, 2.4621472618371403, 1.629468290031214, 1.8469331471633454, 1.452438714650874, 0.9530063641283487, 0.8735199206828114 ], "peak_layer": 22, "peak_normalized": 2.9606390037219725, "population_cv": 0.9681787165444765, "rest_mean": 4.699617262236576e-05, "spike_contrast": 3.7899751783986364, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00017811432771850378, "top_five_layers": [ 22, 26, 25, 21, 23 ], "top_five_spike_overlap": 4 }, "values": [ 3.9256163290701807e-05, 1.4393806850421242e-05, 9.337367373518646e-06, 1.7583412045496516e-05, 1.278224681300344e-05, 1.03036636573961e-05, 1.3893610230297782e-05, 2.432141081953887e-05, 2.775242319330573e-05, 2.0585635866154917e-05, 1.8457110854797065e-05, 1.843657264544163e-05, 2.405131635896396e-05, 2.5390458176843822e-05, 2.8674479835899547e-05, 2.8330305212875828e-05, 2.382393904554192e-05, 3.028337778232526e-05, 2.8044516511727124e-05, 3.602628567023203e-05, 0.00017567329632584006, 0.00019979393982794136, 0.00017368147382512689, 0.00014747587556485087, 0.0001939470530487597, 0.00019513953884597868, 0.00016615403001196682, 0.00010996203491231427, 0.00012463729945011437, 9.801548003451899e-05, 6.431209476431832e-05, 5.894808055018075e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.015726652927696705, "first_to_last_ratio": 0.14031202298106044, "imbalance_abs_log_ratio": 1.9638866007408895, "last_quartile_mean": 0.11208343086764216, "mean": 0.05984494066797197, "normalized": [ 0.5817159394623853, 0.21329409008400638, 0.13836543030820245, 0.26055913499009653, 0.18941324063527437, 0.1526844532232541, 0.20588194088455028, 0.36040590002189726, 0.41124824181954295, 0.3050474835118139, 0.2735060178075916, 0.273201651110741, 0.35640353553867715, 0.37624753453424215, 0.42491172006038797, 0.41981156257536634, 0.35303415045149444, 0.4487531036157151, 0.4155766045421345, 0.5338542063524581, 2.6032081764056025, 2.9606389456555853, 2.5736924190660213, 2.185365706690064, 2.873996992351884, 2.891668001746537, 2.4621473625257404, 1.6294681803958442, 1.8469332209343858, 1.4524387488575696, 0.9530063959374594, 0.8735199079034743 ], "peak_layer": 22, "peak_normalized": 2.9606389456555853, "population_cv": 0.9681787143090018, "rest_mean": 0.04167667665967235, "spike_contrast": 3.789975088527889, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.15795356631278992, "top_five_layers": [ 22, 26, 25, 21, 23 ], "top_five_spike_overlap": 4 }, "values": [ 0.03481275588274002, 0.01276457216590643, 0.008280470967292786, 0.015593145973980427, 0.011335424147546291, 0.009137392044067383, 0.012320992536842823, 0.02156846970319748, 0.024611126631498337, 0.018255548551678658, 0.016367951408028603, 0.016349736601114273, 0.021328948438167572, 0.022516511380672455, 0.02542881667613983, 0.0251235980540514, 0.021127307787537575, 0.026855602860450745, 0.02487015724182129, 0.03194847330451012, 0.1557888388633728, 0.1771792620420456, 0.1540224701166153, 0.1307830810546875, 0.1719941794872284, 0.17305169999599457, 0.14734706282615662, 0.09751542657613754, 0.11052960902452469, 0.08692111074924469, 0.05703261122107506, 0.05227574706077576 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 1.0061295867558329e-06, "first_to_last_ratio": 0.11842320932790659, "imbalance_abs_log_ratio": 2.1334905510136277, "last_quartile_mean": 8.49605066832737e-06, "mean": 4.365163350783519e-06, "normalized": [ 0.5368020226986673, 0.16971060838105761, 0.13410619991648343, 0.2737672720355794, 0.17254528945791953, 0.13905042866169556, 0.15902329960072473, 0.2589205349972531, 0.33151883176756186, 0.2629812299966789, 0.24045074835041885, 0.24352968420582896, 0.32204843448988535, 0.36030617882440874, 0.42811660272336727, 0.41111875160131917, 0.35123001138976506, 0.43809006822116814, 0.40879993942951587, 0.3687203828633752, 2.5744168736304367, 2.993182569498261, 2.6417671760629977, 2.209152675699325, 2.926027285435564, 2.960732429505928, 2.7232927720785614, 1.7159123604866093, 1.8688437451101783, 1.456254776895987, 1.0064626168064281, 0.9131181991770502 ], "peak_layer": 22, "peak_normalized": 2.993182569498261, "population_cv": 1.010043630070163, "rest_mean": 3.0160778355484246e-06, "spike_contrast": 3.8627070547516635, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.1650225133053027e-05, "top_five_layers": [ 22, 26, 25, 27, 23 ], "top_five_spike_overlap": 3 }, "values": [ 2.3432285161106847e-06, 7.408145279441669e-07, 5.853954689882812e-07, 1.1950388625336927e-06, 7.531883738920442e-07, 6.069778351047717e-07, 6.941626793377509e-07, 1.1302304301352706e-06, 1.447133854526328e-06, 1.147956027125474e-06, 1.049606794367719e-06, 1.0630468523231684e-06, 1.4057940234124544e-06, 1.5727953268651618e-06, 1.8687989040699904e-06, 1.7946005073099514e-06, 1.5331763734138804e-06, 1.9123347101412946e-06, 1.784478513400245e-06, 1.6095247019620729e-06, 1.1237750186410267e-05, 1.306573085457785e-05, 1.1531745258253068e-05, 9.643312296248041e-06, 1.277258706977591e-05, 1.2924080692755524e-05, 1.188761780213099e-05, 7.490237749152584e-06, 8.157808224495966e-06, 6.356789981509792e-06, 4.393373728817096e-06, 3.985910097981105e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 1.197550119513835e-05, "first_to_last_ratio": 0.1587346511851631, "imbalance_abs_log_ratio": 1.8405213313314666, "last_quartile_mean": 7.544352229160722e-05, "mean": 4.3035416283032646e-05, "normalized": [ 0.6336653627326162, 0.22557503269458085, 0.11384714614846705, 0.24237874371328197, 0.21264877275758737, 0.17043176216342354, 0.2212795583484355, 0.40634029614109113, 0.4665961306783005, 0.344071525310092, 0.31263674643247324, 0.325989461953546, 0.4165298351149085, 0.4247779117423742, 0.4717272077470243, 0.4737094580054267, 0.40283409959357847, 0.521705164723695, 0.4749365612827932, 0.5725415698804192, 2.7082874349482955, 3.047902384226458, 2.6438885206938516, 2.1412461889756425, 2.9349243481759872, 2.7222959905080586, 2.283569880862303, 1.5378599969243498, 1.615015848446693, 1.3002882037476036, 0.8505045545457767, 0.7799943007808661 ], "peak_layer": 22, "peak_normalized": 3.047902384226458, "population_cv": 0.9449871736776022, "rest_mean": 2.9525086692619544e-05, "spike_contrast": 3.928564114944936, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001159911960712634, "top_five_layers": [ 22, 25, 26, 21, 23 ], "top_five_spike_overlap": 4 }, "values": [ 2.727005266933702e-05, 9.707715435069986e-06, 4.899459327134537e-06, 1.0430870133859571e-05, 9.151428457698785e-06, 7.334601832553744e-06, 9.522857908450533e-06, 1.748702379700262e-05, 2.0080158719792962e-05, 1.4807261322857812e-05, 1.3454452528094407e-05, 1.4029092199052684e-05, 1.7925534848473035e-05, 1.8280494259670377e-05, 2.0300976757425815e-05, 2.038628372247331e-05, 1.733613316901028e-05, 2.245179894089233e-05, 2.0439092622837052e-05, 2.4639564799144864e-05, 0.00011655227717710659, 0.00013116774789523333, 0.00011378084309399128, 9.214942110702395e-05, 0.00012630569108296186, 0.00011715514119714499, 9.827438043430448e-05, 6.61824451526627e-05, 6.95028793415986e-05, 5.5958444136194885e-05, 3.6601817555492744e-05, 3.3567379432497546e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 7.998359336625072e-06, "first_to_last_ratio": 0.19394161321534342, "imbalance_abs_log_ratio": 1.6401981280247118, "last_quartile_mean": 4.124106840208697e-05, "mean": 2.6285412580762113e-05, "normalized": [ 0.6933384670594045, 0.2456346256669455, 0.10397585233667712, 0.2374486790180797, 0.24369297059389394, 0.19793868329883177, 0.24449766268182652, 0.46778427650330523, 0.5456273930041772, 0.3924025527768945, 0.36245298346490396, 0.39321573934626614, 0.49513567631933314, 0.492561727340712, 0.5399874725375117, 0.5520792544136082, 0.47500593515532746, 0.6277771942942818, 0.5578424227128029, 0.6222043300599132, 2.928340543497061, 3.220069339518848, 2.7546600577445735, 2.05455257890152, 2.8717816218189083, 2.429966999218037, 1.946404647300736, 1.3608722565402098, 1.3761877892744612, 1.13757832466242, 0.7380055991332236, 0.6909763438053044 ], "peak_layer": 22, "peak_normalized": 3.220069339518848, "population_cv": 0.9165336328191448, "rest_mean": 1.7689689221461542e-05, "spike_contrast": 4.109869755811259, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 7.27023187209852e-05, "top_five_layers": [ 22, 21, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 1.822468766476959e-05, 6.4566074797767214e-06, 2.7330481771059567e-06, 6.241436494747177e-06, 6.405570275092032e-06, 5.2028999562026e-06, 6.426721938623814e-06, 1.2295902706682682e-05, 1.4342041140480433e-05, 1.0314462997484952e-05, 9.527226211503148e-06, 1.033583794196602e-05, 1.3014845535508357e-05, 1.2947188224643469e-05, 1.4193793504091445e-05, 1.4511630979541223e-05, 1.2485726983868517e-05, 1.6501382560818456e-05, 1.4663118236057926e-05, 1.6354897525161505e-05, 7.697263936279342e-05, 8.464085112791508e-05, 7.24073761375621e-05, 5.4004762205295265e-05, 7.548596477136016e-05, 6.387268513208255e-05, 5.116204920341261e-05, 3.577108873287216e-05, 3.617366382968612e-05, 2.9901715606683865e-05, 1.9398781660129316e-05, 1.8162598280468956e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 3.53738644207624e-05, "first_to_last_ratio": 0.131222819995581, "imbalance_abs_log_ratio": 2.0308584847035727, "last_quartile_mean": 0.0002695709817999159, "mean": 0.00014080404997685037, "normalized": [ 0.577470254414939, 0.20295174191935292, 0.1151700578824124, 0.23719265118106814, 0.1794139596985285, 0.14982370308789797, 0.19985925174978317, 0.3479392297690331, 0.40530357048439714, 0.29950920312948914, 0.2746918219238327, 0.2680798564391957, 0.33955736615799725, 0.36796996435571633, 0.4082064803862017, 0.40241709302556344, 0.34764889743937344, 0.4446227376348821, 0.404675572121426, 0.502974600996141, 2.462986532813244, 2.8953303082983743, 2.595561209030766, 2.254551431946411, 3.008292708807131, 2.942171245358548, 2.6598889524780778, 1.7951160354833269, 1.7746811041390398, 1.3931209151914112, 0.9062455092896525, 0.836576033366786 ], "peak_layer": 25, "peak_normalized": 3.008292708807131, "population_cv": 0.9883689549295365, "rest_mean": 9.795413286526498e-05, "spike_contrast": 3.799672269993553, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0003721936023794115, "top_five_layers": [ 25, 26, 22, 27, 23 ], "top_five_spike_overlap": 3 }, "values": [ 8.131015056278557e-05, 2.8576427212101407e-05, 1.6216410585911945e-05, 3.339768591104075e-05, 2.5262212147936225e-05, 2.1095784177305177e-05, 2.814099207171239e-05, 4.8991252697305754e-05, 5.706838419428095e-05, 4.217210880597122e-05, 3.867772102239542e-05, 3.7746729503851384e-05, 4.7811052354518324e-05, 5.181166125112213e-05, 5.7477125665172935e-05, 5.666195647791028e-05, 4.8950372729450464e-05, 6.260468217078596e-05, 5.697995948139578e-05, 7.082086085574701e-05, 0.0003467984788585454, 0.0004076742334291339, 0.00036546553019434214, 0.00031744997249916196, 0.0004235797969158739, 0.00041426962707191706, 0.00037452313699759543, 0.00025275960797443986, 0.00024988228688016534, 0.000196157066966407, 0.00012760303798131645, 0.00011779329361161217 ] } } } }, "selector": { "expected_uniform_count": 0, "identities_unique": true, "identity_order_sha256": "97fd9721a263024f6a746638f32e0ad9a988cd4c111c00e9e178c1022e5c9d5e", "passed": true, "uniform_count": 0, "uniform_indices": [], "visit_count": 65, "visits": [ { "branch": "attention", "coefficient": "detached_learned", "group": 1, "index": 0, "kind": "depth", "layer": 1, "offset": 1, "sources": 1, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 1, "index": 1, "kind": "depth", "layer": 1, "offset": 1, "sources": 2, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 1, "index": 2, "kind": "depth", "layer": 2, "offset": 2, "sources": 2, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 1, "index": 3, "kind": "depth", "layer": 2, "offset": 2, "sources": 2, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 1, "index": 4, "kind": "depth", "layer": 3, "offset": 3, "sources": 2, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 1, "index": 5, "kind": "depth", "layer": 3, "offset": 3, "sources": 2, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 1, "index": 6, "kind": "depth", "layer": 4, "offset": 4, "sources": 2, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 1, "index": 7, "kind": "depth", "layer": 4, "offset": 4, "sources": 2, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 2, "index": 8, "kind": "depth", "layer": 5, "offset": 1, "sources": 2, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 2, "index": 9, "kind": "depth", "layer": 5, "offset": 1, "sources": 3, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 2, "index": 10, "kind": "depth", "layer": 6, "offset": 2, "sources": 3, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 2, "index": 11, "kind": "depth", "layer": 6, "offset": 2, "sources": 3, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 2, "index": 12, "kind": "depth", "layer": 7, "offset": 3, "sources": 3, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 2, "index": 13, "kind": "depth", "layer": 7, "offset": 3, "sources": 3, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 2, "index": 14, "kind": "depth", "layer": 8, "offset": 4, "sources": 3, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 2, "index": 15, "kind": "depth", "layer": 8, "offset": 4, "sources": 3, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 3, "index": 16, "kind": "depth", "layer": 9, "offset": 1, "sources": 3, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 3, "index": 17, "kind": "depth", "layer": 9, "offset": 1, "sources": 4, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 3, "index": 18, "kind": "depth", "layer": 10, "offset": 2, "sources": 4, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 3, "index": 19, "kind": "depth", "layer": 10, "offset": 2, "sources": 4, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 3, "index": 20, "kind": "depth", "layer": 11, "offset": 3, "sources": 4, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 3, "index": 21, "kind": "depth", "layer": 11, "offset": 3, "sources": 4, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 3, "index": 22, "kind": "depth", "layer": 12, "offset": 4, "sources": 4, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 3, "index": 23, "kind": "depth", "layer": 12, "offset": 4, "sources": 4, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 4, "index": 24, "kind": "depth", "layer": 13, "offset": 1, "sources": 4, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 4, "index": 25, "kind": "depth", "layer": 13, "offset": 1, "sources": 5, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 4, "index": 26, "kind": "depth", "layer": 14, "offset": 2, "sources": 5, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 4, "index": 27, "kind": "depth", "layer": 14, "offset": 2, "sources": 5, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 4, "index": 28, "kind": "depth", "layer": 15, "offset": 3, "sources": 5, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 4, "index": 29, "kind": "depth", "layer": 15, "offset": 3, "sources": 5, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 4, "index": 30, "kind": "depth", "layer": 16, "offset": 4, "sources": 5, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 4, "index": 31, "kind": "depth", "layer": 16, "offset": 4, "sources": 5, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 5, "index": 32, "kind": "depth", "layer": 17, "offset": 1, "sources": 5, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 5, "index": 33, "kind": "depth", "layer": 17, "offset": 1, "sources": 6, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 5, "index": 34, "kind": "depth", "layer": 18, "offset": 2, "sources": 6, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 5, "index": 35, "kind": "depth", "layer": 18, "offset": 2, "sources": 6, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 5, "index": 36, "kind": "depth", "layer": 19, "offset": 3, "sources": 6, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 5, "index": 37, "kind": "depth", "layer": 19, "offset": 3, "sources": 6, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 5, "index": 38, "kind": "depth", "layer": 20, "offset": 4, "sources": 6, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 5, "index": 39, "kind": "depth", "layer": 20, "offset": 4, "sources": 6, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 6, "index": 40, "kind": "depth", "layer": 21, "offset": 1, "sources": 6, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 6, "index": 41, "kind": "depth", "layer": 21, "offset": 1, "sources": 7, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 6, "index": 42, "kind": "depth", "layer": 22, "offset": 2, "sources": 7, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 6, "index": 43, "kind": "depth", "layer": 22, "offset": 2, "sources": 7, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 6, "index": 44, "kind": "depth", "layer": 23, "offset": 3, "sources": 7, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 6, "index": 45, "kind": "depth", "layer": 23, "offset": 3, "sources": 7, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 6, "index": 46, "kind": "depth", "layer": 24, "offset": 4, "sources": 7, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 6, "index": 47, "kind": "depth", "layer": 24, "offset": 4, "sources": 7, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 7, "index": 48, "kind": "depth", "layer": 25, "offset": 1, "sources": 7, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 7, "index": 49, "kind": "depth", "layer": 25, "offset": 1, "sources": 8, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 7, "index": 50, "kind": "depth", "layer": 26, "offset": 2, "sources": 8, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 7, "index": 51, "kind": "depth", "layer": 26, "offset": 2, "sources": 8, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 7, "index": 52, "kind": "depth", "layer": 27, "offset": 3, "sources": 8, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 7, "index": 53, "kind": "depth", "layer": 27, "offset": 3, "sources": 8, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 7, "index": 54, "kind": "depth", "layer": 28, "offset": 4, "sources": 8, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 7, "index": 55, "kind": "depth", "layer": 28, "offset": 4, "sources": 8, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 8, "index": 56, "kind": "depth", "layer": 29, "offset": 1, "sources": 8, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 8, "index": 57, "kind": "depth", "layer": 29, "offset": 1, "sources": 9, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 8, "index": 58, "kind": "depth", "layer": 30, "offset": 2, "sources": 9, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 8, "index": 59, "kind": "depth", "layer": 30, "offset": 2, "sources": 9, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 8, "index": 60, "kind": "depth", "layer": 31, "offset": 3, "sources": 9, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 8, "index": 61, "kind": "depth", "layer": 31, "offset": 3, "sources": 9, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 8, "index": 62, "kind": "depth", "layer": 32, "offset": 4, "sources": 9, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 8, "index": 63, "kind": "depth", "layer": 32, "offset": 4, "sources": 9, "use_uniform": false }, { "branch": "output", "coefficient": "detached_learned", "group": null, "index": 64, "kind": "output", "layer": null, "offset": null, "sources": 9, "use_uniform": false } ], "visits_sha256": "5675dee7ec78bda7aa686371971870d4075869f1cf0f2f1b2e5e04bb1a176220" } }, "uniform_all": { "bits_per_byte": 1.8740129107275643, "forward": { "activation_sha256": { "attention_branch_output": "5e6ecbcd78543fc42d4098a51169dca446d404ce384015643e84d2c78433c949", "mlp_branch_output": "c0b8018c349b1bd09e5d17858eb312a2323da0e92de9651e2bb08ae4adf115c7", "post_attention_state": "90a633f12d984935028fe5d7212be48e7da7b4a0806751939a551938d16dc540", "post_mlp_state": "89f2226456bfe96e245d2aad0f699d32c676f9f53099789cd870253d002402f5", "pre_attention_input": "ee695e78937de01b7128d3866b8fc5dd328ebdf2c159be37918d4f02952d5248", "pre_mlp_input": "e53fa425d89dfeb0e9395eca9bcd73dc0a55314abf45c5b10b43e2cbfa724345" }, "logits_sha256": "bdacba5675f0579cec629bbf01b36c8660ad576cfdf9fef1bea8259eaeac9ec2", "loss_nats": 1.2989667654037476, "mixer_summary_sha256": "eb3ac0f07de9b755d6e67a510ab68d09d9ad666bb7b5d020ba7d8c5055c399b2" }, "loss_nats": 1.2989667654037476, "loss_scale": 1.0, "mixers": [ { "branch": "attention", "entropy_mean": -0.0, "group": 1, "latest_source_mass": 1.0, "layer": 1, "max_source_mass": 1.0, "mixer_index": 0, "normalized_entropy": 1.0, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 1.0, "median": 1.0, "p05": 1.0, "p95": 1.0 } ], "sources": 1 }, { "branch": "mlp", "entropy_mean": 0.22886337339878082, "group": 1, "latest_source_mass": 0.06350618600845337, "layer": 1, "max_source_mass": 0.9364938139915466, "mixer_index": 1, "normalized_entropy": 0.3301800538435402, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.9364938139915466, "median": 0.9452826976776123, "p05": 0.8725647330284119, "p95": 0.9712384939193726 }, { "label": "current_group_partial", "mean": 0.06350618600845337, "median": 0.0547172985970974, "p05": 0.028761589899659157, "p95": 0.12743529677391052 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.603729784488678, "group": 1, "latest_source_mass": 0.4133155345916748, "layer": 2, "max_source_mass": 0.5866844654083252, "mixer_index": 2, "normalized_entropy": 0.8709979661187783, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.5866844654083252, "median": 0.5734395384788513, "p05": 0.29813364148139954, "p95": 0.8843642473220825 }, { "label": "current_group_partial", "mean": 0.4133155345916748, "median": 0.4265604615211487, "p05": 0.11563576757907867, "p95": 0.7018663287162781 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.3967982828617096, "group": 1, "latest_source_mass": 0.14110636711120605, "layer": 2, "max_source_mass": 0.858893632888794, "mixer_index": 3, "normalized_entropy": 0.5724589149178446, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.858893632888794, "median": 0.8705973029136658, "p05": 0.7599387168884277, "p95": 0.9270997643470764 }, { "label": "current_group_partial", "mean": 0.14110636711120605, "median": 0.12940272688865662, "p05": 0.07290027290582657, "p95": 0.2400612235069275 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6852496862411499, "group": 1, "latest_source_mass": 0.518975019454956, "layer": 3, "max_source_mass": 0.518975019454956, "mixer_index": 4, "normalized_entropy": 0.9886063241108252, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.48102498054504395, "median": 0.48102718591690063, "p05": 0.38786420226097107, "p95": 0.580143928527832 }, { "label": "current_group_partial", "mean": 0.518975019454956, "median": 0.5189728140830994, "p05": 0.41985607147216797, "p95": 0.6121357679367065 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.5890734195709229, "group": 1, "latest_source_mass": 0.3123423159122467, "layer": 3, "max_source_mass": 0.6876576542854309, "mixer_index": 5, "normalized_entropy": 0.8498533011344741, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.6876576542854309, "median": 0.6909762620925903, "p05": 0.48803940415382385, "p95": 0.8900076746940613 }, { "label": "current_group_partial", "mean": 0.3123423159122467, "median": 0.3090237081050873, "p05": 0.10999235510826111, "p95": 0.5119606256484985 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6873323321342468, "group": 1, "latest_source_mass": 0.4831485152244568, "layer": 4, "max_source_mass": 0.5168514251708984, "mixer_index": 6, "normalized_entropy": 0.9916109470127238, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.5168514251708984, "median": 0.5219584703445435, "p05": 0.4251725673675537, "p95": 0.5947866439819336 }, { "label": "current_group_partial", "mean": 0.4831485152244568, "median": 0.47804147005081177, "p05": 0.405213326215744, "p95": 0.5748274326324463 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.487301766872406, "group": 1, "latest_source_mass": 0.2060384750366211, "layer": 4, "max_source_mass": 0.7939615249633789, "mixer_index": 7, "normalized_entropy": 0.7030278424832499, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.7939615249633789, "median": 0.7953404188156128, "p05": 0.6590759754180908, "p95": 0.9329437017440796 }, { "label": "current_group_partial", "mean": 0.2060384750366211, "median": 0.20465964078903198, "p05": 0.06705628335475922, "p95": 0.3409239649772644 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.674697756767273, "group": 2, "latest_source_mass": 0.42952626943588257, "layer": 5, "max_source_mass": 0.5704737305641174, "mixer_index": 8, "normalized_entropy": 0.9733831077870527, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.5704737305641174, "median": 0.5696132779121399, "p05": 0.4695202708244324, "p95": 0.6750621199607849 }, { "label": "completed_group_1", "mean": 0.42952626943588257, "median": 0.4303867816925049, "p05": 0.3249378800392151, "p95": 0.5304797887802124 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.9691112041473389, "group": 2, "latest_source_mass": 0.2681158185005188, "layer": 5, "max_source_mass": 0.5506307482719421, "mixer_index": 9, "normalized_entropy": 0.8821230329784768, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.5506307482719421, "median": 0.543993353843689, "p05": 0.41262108087539673, "p95": 0.7161091566085815 }, { "label": "completed_group_1", "mean": 0.18125343322753906, "median": 0.1794438362121582, "p05": 0.11575750261545181, "p95": 0.25031501054763794 }, { "label": "current_group_partial", "mean": 0.2681158185005188, "median": 0.2664111256599426, "p05": 0.1564716249704361, "p95": 0.3817881643772125 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0745112895965576, "group": 2, "latest_source_mass": 0.412794291973114, "layer": 6, "max_source_mass": 0.412794291973114, "mixer_index": 10, "normalized_entropy": 0.9780623252441762, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.34262996912002563, "median": 0.3412924110889435, "p05": 0.29556700587272644, "p95": 0.39125555753707886 }, { "label": "completed_group_1", "mean": 0.24457570910453796, "median": 0.24382859468460083, "p05": 0.22120657563209534, "p95": 0.2734297513961792 }, { "label": "current_group_partial", "mean": 0.412794291973114, "median": 0.4151499271392822, "p05": 0.37735897302627563, "p95": 0.43749260902404785 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0074045658111572, "group": 2, "latest_source_mass": 0.3775783181190491, "layer": 6, "max_source_mass": 0.4399106502532959, "mixer_index": 11, "normalized_entropy": 0.9169791528842925, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.4399106502532959, "median": 0.44856905937194824, "p05": 0.22063353657722473, "p95": 0.6359403133392334 }, { "label": "completed_group_1", "mean": 0.18251106142997742, "median": 0.18562838435173035, "p05": 0.1196163073182106, "p95": 0.23045453429222107 }, { "label": "current_group_partial", "mean": 0.3775783181190491, "median": 0.3675365447998047, "p05": 0.23817461729049683, "p95": 0.5557451844215393 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.060497522354126, "group": 2, "latest_source_mass": 0.4025024175643921, "layer": 7, "max_source_mass": 0.4025024175643921, "mixer_index": 12, "normalized_entropy": 0.9653064445872968, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.3786153197288513, "median": 0.3907395303249359, "p05": 0.28039106726646423, "p95": 0.4436405599117279 }, { "label": "completed_group_1", "mean": 0.2188822478055954, "median": 0.21226292848587036, "p05": 0.1885872334241867, "p95": 0.2672030031681061 }, { "label": "current_group_partial", "mean": 0.4025024175643921, "median": 0.39815032482147217, "p05": 0.3533596992492676, "p95": 0.460100919008255 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 0.9941019415855408, "group": 2, "latest_source_mass": 0.4801652431488037, "layer": 7, "max_source_mass": 0.4801652431488037, "mixer_index": 13, "normalized_entropy": 0.9048705824970601, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.32296955585479736, "median": 0.2803739905357361, "p05": 0.1784922033548355, "p95": 0.6440245509147644 }, { "label": "completed_group_1", "mean": 0.19686520099639893, "median": 0.2053825855255127, "p05": 0.09813719242811203, "p95": 0.2504696547985077 }, { "label": "current_group_partial", "mean": 0.4801652431488037, "median": 0.5057140588760376, "p05": 0.2529003322124481, "p95": 0.598870575428009 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0503522157669067, "group": 2, "latest_source_mass": 0.4001518487930298, "layer": 8, "max_source_mass": 0.40130966901779175, "mixer_index": 14, "normalized_entropy": 0.9560717885654542, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.40130966901779175, "median": 0.396759569644928, "p05": 0.3479611873626709, "p95": 0.4733521044254303 }, { "label": "completed_group_1", "mean": 0.19853851199150085, "median": 0.19769755005836487, "p05": 0.17136667668819427, "p95": 0.22788912057876587 }, { "label": "current_group_partial", "mean": 0.4001518487930298, "median": 0.40291696786880493, "p05": 0.3506356179714203, "p95": 0.43941178917884827 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0129402875900269, "group": 2, "latest_source_mass": 0.42161279916763306, "layer": 8, "max_source_mass": 0.42161279916763306, "mixer_index": 15, "normalized_entropy": 0.9220179839951123, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.4093267023563385, "median": 0.4046899676322937, "p05": 0.2895161807537079, "p95": 0.5762131810188293 }, { "label": "completed_group_1", "mean": 0.16906049847602844, "median": 0.16888955235481262, "p05": 0.11233551055192947, "p95": 0.22477403283119202 }, { "label": "current_group_partial", "mean": 0.42161279916763306, "median": 0.4227708578109741, "p05": 0.3131049573421478, "p95": 0.519075334072113 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0774019956588745, "group": 3, "latest_source_mass": 0.3560434877872467, "layer": 9, "max_source_mass": 0.38510820269584656, "mixer_index": 16, "normalized_entropy": 0.9806935592947451, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.38510820269584656, "median": 0.3892287313938141, "p05": 0.2934730052947998, "p95": 0.4718902111053467 }, { "label": "completed_group_1", "mean": 0.25884830951690674, "median": 0.25734007358551025, "p05": 0.22217319905757904, "p95": 0.29551801085472107 }, { "label": "completed_group_2", "mean": 0.3560434877872467, "median": 0.35412120819091797, "p05": 0.29444968700408936, "p95": 0.41908568143844604 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.2740377187728882, "group": 3, "latest_source_mass": 0.3633861243724823, "layer": 9, "max_source_mass": 0.3633861243724823, "mixer_index": 17, "normalized_entropy": 0.9190239493895668, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.29550763964653015, "median": 0.30360478162765503, "p05": 0.16904757916927338, "p95": 0.45146965980529785 }, { "label": "completed_group_1", "mean": 0.09541244804859161, "median": 0.09486871212720871, "p05": 0.061360105872154236, "p95": 0.13037073612213135 }, { "label": "completed_group_2", "mean": 0.24569380283355713, "median": 0.24333015084266663, "p05": 0.1698218435049057, "p95": 0.31589236855506897 }, { "label": "current_group_partial", "mean": 0.3633861243724823, "median": 0.36472851037979126, "p05": 0.2795379161834717, "p95": 0.4399876892566681 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.360302448272705, "group": 3, "latest_source_mass": 0.28569501638412476, "layer": 10, "max_source_mass": 0.3109527826309204, "mixer_index": 18, "normalized_entropy": 0.9812507981160736, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.3109527826309204, "median": 0.32172977924346924, "p05": 0.23520725965499878, "p95": 0.37052470445632935 }, { "label": "completed_group_1", "mean": 0.20984819531440735, "median": 0.20958667993545532, "p05": 0.1891367882490158, "p95": 0.23184512555599213 }, { "label": "completed_group_2", "mean": 0.19350400567054749, "median": 0.19153150916099548, "p05": 0.16607457399368286, "p95": 0.22877082228660583 }, { "label": "current_group_partial", "mean": 0.28569501638412476, "median": 0.28060320019721985, "p05": 0.23824527859687805, "p95": 0.3439165949821472 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.2722704410552979, "group": 3, "latest_source_mass": 0.29912811517715454, "layer": 10, "max_source_mass": 0.35036465525627136, "mixer_index": 19, "normalized_entropy": 0.9177491279900463, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.35036465525627136, "median": 0.36210548877716064, "p05": 0.12807878851890564, "p95": 0.5797606706619263 }, { "label": "completed_group_1", "mean": 0.12631990015506744, "median": 0.1264108121395111, "p05": 0.06977735459804535, "p95": 0.1755298376083374 }, { "label": "completed_group_2", "mean": 0.22418735921382904, "median": 0.22563719749450684, "p05": 0.1176326721906662, "p95": 0.32001155614852905 }, { "label": "current_group_partial", "mean": 0.29912811517715454, "median": 0.28589946031570435, "p05": 0.193109393119812, "p95": 0.44119665026664734 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3558036088943481, "group": 3, "latest_source_mass": 0.314197838306427, "layer": 11, "max_source_mass": 0.314197838306427, "mixer_index": 20, "normalized_entropy": 0.978005571485618, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.298976868391037, "median": 0.2961939573287964, "p05": 0.24950163066387177, "p95": 0.35835936665534973 }, { "label": "completed_group_1", "mean": 0.1941717267036438, "median": 0.1918032318353653, "p05": 0.16809208691120148, "p95": 0.22895419597625732 }, { "label": "completed_group_2", "mean": 0.1926535815000534, "median": 0.19306811690330505, "p05": 0.16983944177627563, "p95": 0.2155935913324356 }, { "label": "current_group_partial", "mean": 0.314197838306427, "median": 0.31541913747787476, "p05": 0.2673022747039795, "p95": 0.35941022634506226 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.2696614265441895, "group": 3, "latest_source_mass": 0.2862822115421295, "layer": 11, "max_source_mass": 0.3667730689048767, "mixer_index": 21, "normalized_entropy": 0.9158671218416545, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.3667730689048767, "median": 0.4067659378051758, "p05": 0.10593366622924805, "p95": 0.5672274827957153 }, { "label": "completed_group_1", "mean": 0.15310388803482056, "median": 0.15050218999385834, "p05": 0.08651046454906464, "p95": 0.22459912300109863 }, { "label": "completed_group_2", "mean": 0.19384080171585083, "median": 0.18732509016990662, "p05": 0.1267009675502777, "p95": 0.281107634305954 }, { "label": "current_group_partial", "mean": 0.2862822115421295, "median": 0.26271265745162964, "p05": 0.17024393379688263, "p95": 0.475044310092926 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3606936931610107, "group": 3, "latest_source_mass": 0.3028184771537781, "layer": 12, "max_source_mass": 0.3028184771537781, "mixer_index": 22, "normalized_entropy": 0.9815330216461395, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.298227459192276, "median": 0.2954302132129669, "p05": 0.25287362933158875, "p95": 0.35405808687210083 }, { "label": "completed_group_1", "mean": 0.18611131608486176, "median": 0.1850002110004425, "p05": 0.15901757776737213, "p95": 0.21725836396217346 }, { "label": "completed_group_2", "mean": 0.21284276247024536, "median": 0.21402311325073242, "p05": 0.18819917738437653, "p95": 0.23418807983398438 }, { "label": "current_group_partial", "mean": 0.3028184771537781, "median": 0.3041122555732727, "p05": 0.25661808252334595, "p95": 0.34458184242248535 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.269316554069519, "group": 3, "latest_source_mass": 0.2653314769268036, "layer": 12, "max_source_mass": 0.4081852436065674, "mixer_index": 23, "normalized_entropy": 0.9156183489371815, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.4081852436065674, "median": 0.4254342317581177, "p05": 0.20765496790409088, "p95": 0.587238073348999 }, { "label": "completed_group_1", "mean": 0.14420127868652344, "median": 0.1438048928976059, "p05": 0.1064920499920845, "p95": 0.1871720254421234 }, { "label": "completed_group_2", "mean": 0.18228203058242798, "median": 0.17507661879062653, "p05": 0.12183455377817154, "p95": 0.26505547761917114 }, { "label": "current_group_partial", "mean": 0.2653314769268036, "median": 0.24924281239509583, "p05": 0.15609969198703766, "p95": 0.4061311185359955 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3636136054992676, "group": 4, "latest_source_mass": 0.3136237859725952, "layer": 13, "max_source_mass": 0.3136237859725952, "mixer_index": 24, "normalized_entropy": 0.9836392931712563, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.27427661418914795, "median": 0.27403637766838074, "p05": 0.22928935289382935, "p95": 0.319773405790329 }, { "label": "completed_group_1", "mean": 0.21341991424560547, "median": 0.20697325468063354, "p05": 0.18552586436271667, "p95": 0.26688331365585327 }, { "label": "completed_group_2", "mean": 0.19867971539497375, "median": 0.197545126080513, "p05": 0.1794913411140442, "p95": 0.222100168466568 }, { "label": "completed_group_3", "mean": 0.3136237859725952, "median": 0.31875866651535034, "p05": 0.24156072735786438, "p95": 0.3695686459541321 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.570773959159851, "group": 4, "latest_source_mass": 0.2294243425130844, "layer": 13, "max_source_mass": 0.2658740282058716, "mixer_index": 25, "normalized_entropy": 0.9759767351225285, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.2658740282058716, "median": 0.2747163474559784, "p05": 0.17927880585193634, "p95": 0.3512483835220337 }, { "label": "completed_group_1", "mean": 0.13419361412525177, "median": 0.13374894857406616, "p05": 0.11072897166013718, "p95": 0.1586270034313202 }, { "label": "completed_group_2", "mean": 0.1907704919576645, "median": 0.19074805080890656, "p05": 0.14699944853782654, "p95": 0.23316305875778198 }, { "label": "completed_group_3", "mean": 0.17973749339580536, "median": 0.17616114020347595, "p05": 0.1363743096590042, "p95": 0.23202598094940186 }, { "label": "current_group_partial", "mean": 0.2294243425130844, "median": 0.22995705902576447, "p05": 0.17975632846355438, "p95": 0.2789157032966614 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.5560038089752197, "group": 4, "latest_source_mass": 0.32468360662460327, "layer": 14, "max_source_mass": 0.32468360662460327, "mixer_index": 26, "normalized_entropy": 0.9667995248241249, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.20477107167243958, "median": 0.206403911113739, "p05": 0.16619175672531128, "p95": 0.2431035190820694 }, { "label": "completed_group_1", "mean": 0.15721669793128967, "median": 0.15658551454544067, "p05": 0.13878680765628815, "p95": 0.17778633534908295 }, { "label": "completed_group_2", "mean": 0.13811248540878296, "median": 0.13743364810943604, "p05": 0.12352626770734787, "p95": 0.15553249418735504 }, { "label": "completed_group_3", "mean": 0.1752161681652069, "median": 0.17309701442718506, "p05": 0.15223677456378937, "p95": 0.20474471151828766 }, { "label": "current_group_partial", "mean": 0.32468360662460327, "median": 0.32497870922088623, "p05": 0.2946181893348694, "p95": 0.35203880071640015 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.5659973621368408, "group": 4, "latest_source_mass": 0.26703086495399475, "layer": 14, "max_source_mass": 0.26703086495399475, "mixer_index": 27, "normalized_entropy": 0.9730088685238187, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.20531430840492249, "median": 0.20229148864746094, "p05": 0.10698921978473663, "p95": 0.3393132984638214 }, { "label": "completed_group_1", "mean": 0.15986156463623047, "median": 0.15952220559120178, "p05": 0.11969029903411865, "p95": 0.20180393755435944 }, { "label": "completed_group_2", "mean": 0.21557100117206573, "median": 0.21654027700424194, "p05": 0.1673085242509842, "p95": 0.2610440254211426 }, { "label": "completed_group_3", "mean": 0.15222224593162537, "median": 0.1510905921459198, "p05": 0.11800260841846466, "p95": 0.18893754482269287 }, { "label": "current_group_partial", "mean": 0.26703086495399475, "median": 0.258539617061615, "p05": 0.19330690801143646, "p95": 0.36521556973457336 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.5552496910095215, "group": 4, "latest_source_mass": 0.31774693727493286, "layer": 15, "max_source_mass": 0.31774693727493286, "mixer_index": 28, "normalized_entropy": 0.9663309649872576, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.2162160575389862, "median": 0.2145705670118332, "p05": 0.17727568745613098, "p95": 0.26878347992897034 }, { "label": "completed_group_1", "mean": 0.17982497811317444, "median": 0.18024154007434845, "p05": 0.15453973412513733, "p95": 0.2030555009841919 }, { "label": "completed_group_2", "mean": 0.1307673156261444, "median": 0.13107559084892273, "p05": 0.11463287472724915, "p95": 0.14630235731601715 }, { "label": "completed_group_3", "mean": 0.1554446816444397, "median": 0.15516987442970276, "p05": 0.13627474009990692, "p95": 0.1754152774810791 }, { "label": "current_group_partial", "mean": 0.31774693727493286, "median": 0.319039523601532, "p05": 0.27606821060180664, "p95": 0.35650834441185 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.5068919658660889, "group": 4, "latest_source_mass": 0.3298327624797821, "layer": 15, "max_source_mass": 0.3298327624797821, "mixer_index": 29, "normalized_entropy": 0.9362846209998111, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.20646697282791138, "median": 0.18775638937950134, "p05": 0.0806753933429718, "p95": 0.4274584949016571 }, { "label": "completed_group_1", "mean": 0.1197461187839508, "median": 0.11989086866378784, "p05": 0.0809398666024208, "p95": 0.15743419528007507 }, { "label": "completed_group_2", "mean": 0.19215428829193115, "median": 0.19620640575885773, "p05": 0.1148109883069992, "p95": 0.2491212636232376 }, { "label": "completed_group_3", "mean": 0.15179988741874695, "median": 0.15212899446487427, "p05": 0.10697861760854721, "p95": 0.19542087614536285 }, { "label": "current_group_partial", "mean": 0.3298327624797821, "median": 0.32837340235710144, "p05": 0.22746729850769043, "p95": 0.4339301884174347 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.5435223579406738, "group": 4, "latest_source_mass": 0.3224271535873413, "layer": 16, "max_source_mass": 0.3224271535873413, "mixer_index": 30, "normalized_entropy": 0.9590443632623663, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.22410975396633148, "median": 0.22760415077209473, "p05": 0.14593973755836487, "p95": 0.30093055963516235 }, { "label": "completed_group_1", "mean": 0.16986176371574402, "median": 0.16963240504264832, "p05": 0.13569067418575287, "p95": 0.20651353895664215 }, { "label": "completed_group_2", "mean": 0.13080403208732605, "median": 0.13075244426727295, "p05": 0.10945598781108856, "p95": 0.15187400579452515 }, { "label": "completed_group_3", "mean": 0.15279728174209595, "median": 0.15036612749099731, "p05": 0.12810495495796204, "p95": 0.18468116223812103 }, { "label": "current_group_partial", "mean": 0.3224271535873413, "median": 0.3164973258972168, "p05": 0.2600978910923004, "p95": 0.40530598163604736 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.4949920177459717, "group": 4, "latest_source_mass": 0.31847020983695984, "layer": 16, "max_source_mass": 0.31847020983695984, "mixer_index": 31, "normalized_entropy": 0.9288907675133354, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.27174079418182373, "median": 0.2585451304912567, "p05": 0.13639721274375916, "p95": 0.4698348045349121 }, { "label": "completed_group_1", "mean": 0.12178683280944824, "median": 0.11714950203895569, "p05": 0.08594125509262085, "p95": 0.1723237782716751 }, { "label": "completed_group_2", "mean": 0.1529826819896698, "median": 0.15223106741905212, "p05": 0.10371473431587219, "p95": 0.203575000166893 }, { "label": "completed_group_3", "mean": 0.1350194811820984, "median": 0.13363957405090332, "p05": 0.08964549005031586, "p95": 0.18468207120895386 }, { "label": "current_group_partial", "mean": 0.31847020983695984, "median": 0.3185216188430786, "p05": 0.21444711089134216, "p95": 0.4301546812057495 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.544264793395996, "group": 5, "latest_source_mass": 0.2861037850379944, "layer": 17, "max_source_mass": 0.2861037850379944, "mixer_index": 32, "normalized_entropy": 0.9595056643474137, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.24016115069389343, "median": 0.23868846893310547, "p05": 0.18435582518577576, "p95": 0.2983453571796417 }, { "label": "completed_group_1", "mean": 0.20730282366275787, "median": 0.1998077780008316, "p05": 0.16255050897598267, "p95": 0.28107208013534546 }, { "label": "completed_group_2", "mean": 0.09692573547363281, "median": 0.0967404842376709, "p05": 0.08098706603050232, "p95": 0.11221462488174438 }, { "label": "completed_group_3", "mean": 0.1695065200328827, "median": 0.1696585714817047, "p05": 0.136396586894989, "p95": 0.20130634307861328 }, { "label": "completed_group_4", "mean": 0.2861037850379944, "median": 0.28883200883865356, "p05": 0.22401469945907593, "p95": 0.3409370183944702 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.686682939529419, "group": 5, "latest_source_mass": 0.12281867861747742, "layer": 17, "max_source_mass": 0.2885604202747345, "mixer_index": 33, "normalized_entropy": 0.9413556721740635, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.16755425930023193, "median": 0.1327776312828064, "p05": 0.0790192037820816, "p95": 0.3667412996292114 }, { "label": "completed_group_1", "mean": 0.10962115228176117, "median": 0.10525335371494293, "p05": 0.07840172946453094, "p95": 0.15405339002609253 }, { "label": "completed_group_2", "mean": 0.1783454418182373, "median": 0.17896884679794312, "p05": 0.10802385956048965, "p95": 0.24048911035060883 }, { "label": "completed_group_3", "mean": 0.13310004770755768, "median": 0.13033035397529602, "p05": 0.0904831737279892, "p95": 0.17934636771678925 }, { "label": "completed_group_4", "mean": 0.2885604202747345, "median": 0.2831825315952301, "p05": 0.16967125236988068, "p95": 0.4189499616622925 }, { "label": "current_group_partial", "mean": 0.12281867861747742, "median": 0.12062528729438782, "p05": 0.07461707293987274, "p95": 0.1782219111919403 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.7070834636688232, "group": 5, "latest_source_mass": 0.2988377809524536, "layer": 18, "max_source_mass": 0.2988377809524536, "mixer_index": 34, "normalized_entropy": 0.9527414214834803, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.181177020072937, "median": 0.18419909477233887, "p05": 0.13624237477779388, "p95": 0.22192169725894928 }, { "label": "completed_group_1", "mean": 0.14557164907455444, "median": 0.144476979970932, "p05": 0.1293315887451172, "p95": 0.16486899554729462 }, { "label": "completed_group_2", "mean": 0.08318156749010086, "median": 0.0820261612534523, "p05": 0.07088497281074524, "p95": 0.0990293100476265 }, { "label": "completed_group_3", "mean": 0.11424209177494049, "median": 0.11267061531543732, "p05": 0.0979941189289093, "p95": 0.1353992223739624 }, { "label": "completed_group_4", "mean": 0.17698991298675537, "median": 0.17597860097885132, "p05": 0.14837387204170227, "p95": 0.21050144731998444 }, { "label": "current_group_partial", "mean": 0.2988377809524536, "median": 0.2989804744720459, "p05": 0.2721370458602905, "p95": 0.32572072744369507 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.7419235706329346, "group": 5, "latest_source_mass": 0.26016175746917725, "layer": 18, "max_source_mass": 0.26016175746917725, "mixer_index": 35, "normalized_entropy": 0.972186055410333, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1532188355922699, "median": 0.1451343297958374, "p05": 0.11514341086149216, "p95": 0.22103679180145264 }, { "label": "completed_group_1", "mean": 0.11420352011919022, "median": 0.11261284351348877, "p05": 0.0928458571434021, "p95": 0.14123988151550293 }, { "label": "completed_group_2", "mean": 0.16597424447536469, "median": 0.1655328869819641, "p05": 0.13871800899505615, "p95": 0.19611573219299316 }, { "label": "completed_group_3", "mean": 0.1250365972518921, "median": 0.12369821965694427, "p05": 0.10375828295946121, "p95": 0.14912742376327515 }, { "label": "completed_group_4", "mean": 0.18140508234500885, "median": 0.1796584278345108, "p05": 0.14511024951934814, "p95": 0.22286373376846313 }, { "label": "current_group_partial", "mean": 0.26016175746917725, "median": 0.26266711950302124, "p05": 0.1889389455318451, "p95": 0.3234381675720215 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.7127439975738525, "group": 5, "latest_source_mass": 0.2907913327217102, "layer": 19, "max_source_mass": 0.2907913327217102, "mixer_index": 36, "normalized_entropy": 0.9559006256078307, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.18731768429279327, "median": 0.19042012095451355, "p05": 0.13879702985286713, "p95": 0.23792000114917755 }, { "label": "completed_group_1", "mean": 0.16715484857559204, "median": 0.1667211651802063, "p05": 0.14574480056762695, "p95": 0.19026139378547668 }, { "label": "completed_group_2", "mean": 0.08496320247650146, "median": 0.08452782034873962, "p05": 0.07410907745361328, "p95": 0.097386933863163 }, { "label": "completed_group_3", "mean": 0.11362075805664062, "median": 0.11248823255300522, "p05": 0.0977015346288681, "p95": 0.13219426572322845 }, { "label": "completed_group_4", "mean": 0.15615218877792358, "median": 0.15540224313735962, "p05": 0.1289147436618805, "p95": 0.18598109483718872 }, { "label": "current_group_partial", "mean": 0.2907913327217102, "median": 0.2900415062904358, "p05": 0.25822532176971436, "p95": 0.32475030422210693 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.6076834201812744, "group": 5, "latest_source_mass": 0.3738824725151062, "layer": 19, "max_source_mass": 0.3738824725151062, "mixer_index": 37, "normalized_entropy": 0.8972652009334232, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.15032559633255005, "median": 0.12544110417366028, "p05": 0.06976518034934998, "p95": 0.35840943455696106 }, { "label": "completed_group_1", "mean": 0.08624927699565887, "median": 0.08602266013622284, "p05": 0.061998721212148666, "p95": 0.11138717085123062 }, { "label": "completed_group_2", "mean": 0.15736976265907288, "median": 0.15802210569381714, "p05": 0.10121746361255646, "p95": 0.20687000453472137 }, { "label": "completed_group_3", "mean": 0.08584291487932205, "median": 0.0827617198228836, "p05": 0.060561805963516235, "p95": 0.11351694166660309 }, { "label": "completed_group_4", "mean": 0.14632999897003174, "median": 0.1426895558834076, "p05": 0.10106660425662994, "p95": 0.20179018378257751 }, { "label": "current_group_partial", "mean": 0.3738824725151062, "median": 0.3799295425415039, "p05": 0.23631010949611664, "p95": 0.4884527325630188 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.709290862083435, "group": 5, "latest_source_mass": 0.2901708781719208, "layer": 20, "max_source_mass": 0.2901708781719208, "mixer_index": 38, "normalized_entropy": 0.9539733939957076, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.18991519510746002, "median": 0.1924753338098526, "p05": 0.14201581478118896, "p95": 0.23709386587142944 }, { "label": "completed_group_1", "mean": 0.17091524600982666, "median": 0.17006167769432068, "p05": 0.15297070145606995, "p95": 0.19153237342834473 }, { "label": "completed_group_2", "mean": 0.07822999358177185, "median": 0.07778562605381012, "p05": 0.06707640737295151, "p95": 0.09089364111423492 }, { "label": "completed_group_3", "mean": 0.116765096783638, "median": 0.11516137421131134, "p05": 0.10162825882434845, "p95": 0.13653048872947693 }, { "label": "completed_group_4", "mean": 0.1540035903453827, "median": 0.15287795662879944, "p05": 0.12593276798725128, "p95": 0.18590238690376282 }, { "label": "current_group_partial", "mean": 0.2901708781719208, "median": 0.29104921221733093, "p05": 0.25643545389175415, "p95": 0.3205650746822357 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.6690887212753296, "group": 5, "latest_source_mass": 0.3342471718788147, "layer": 20, "max_source_mass": 0.3342471718788147, "mixer_index": 39, "normalized_entropy": 0.9315361520005944, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.19029226899147034, "median": 0.19644704461097717, "p05": 0.09456793963909149, "p95": 0.30468082427978516 }, { "label": "completed_group_1", "mean": 0.1100241169333458, "median": 0.10808776319026947, "p05": 0.09035106748342514, "p95": 0.1359245330095291 }, { "label": "completed_group_2", "mean": 0.11471512168645859, "median": 0.11494918912649155, "p05": 0.08799737691879272, "p95": 0.14024731516838074 }, { "label": "completed_group_3", "mean": 0.09999909996986389, "median": 0.09682698547840118, "p05": 0.07921088486909866, "p95": 0.1264391839504242 }, { "label": "completed_group_4", "mean": 0.1507222205400467, "median": 0.1476232260465622, "p05": 0.11397356539964676, "p95": 0.19991227984428406 }, { "label": "current_group_partial", "mean": 0.3342471718788147, "median": 0.33802270889282227, "p05": 0.2548266053199768, "p95": 0.40130624175071716 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.7458693981170654, "group": 6, "latest_source_mass": 0.2361394464969635, "layer": 21, "max_source_mass": 0.2361394464969635, "mixer_index": 40, "normalized_entropy": 0.9743882636597644, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.20528709888458252, "median": 0.2140858769416809, "p05": 0.13168928027153015, "p95": 0.27098023891448975 }, { "label": "completed_group_1", "mean": 0.1722964197397232, "median": 0.17238952219486237, "p05": 0.15053534507751465, "p95": 0.19365043938159943 }, { "label": "completed_group_2", "mean": 0.10435444861650467, "median": 0.10361101478338242, "p05": 0.08888213336467743, "p95": 0.12212536484003067 }, { "label": "completed_group_3", "mean": 0.12075909972190857, "median": 0.1180175170302391, "p05": 0.10358668118715286, "p95": 0.14458929002285004 }, { "label": "completed_group_4", "mean": 0.16116347908973694, "median": 0.15841388702392578, "p05": 0.1337081342935562, "p95": 0.19791515171527863 }, { "label": "completed_group_5", "mean": 0.2361394464969635, "median": 0.23709020018577576, "p05": 0.20446214079856873, "p95": 0.26509594917297363 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.4612125158309937, "group": 6, "latest_source_mass": 0.5543455481529236, "layer": 21, "max_source_mass": 0.5543455481529236, "mixer_index": 41, "normalized_entropy": 0.7509146897354808, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.08934466540813446, "median": 0.07911523431539536, "p05": 0.04488275945186615, "p95": 0.1737232506275177 }, { "label": "completed_group_1", "mean": 0.054379627108573914, "median": 0.05344422906637192, "p05": 0.04221750423312187, "p95": 0.06924191862344742 }, { "label": "completed_group_2", "mean": 0.07567165046930313, "median": 0.07438549399375916, "p05": 0.05811355262994766, "p95": 0.09632256627082825 }, { "label": "completed_group_3", "mean": 0.05758609622716904, "median": 0.05612001568078995, "p05": 0.044792283326387405, "p95": 0.07347827404737473 }, { "label": "completed_group_4", "mean": 0.07884950935840607, "median": 0.07757881283760071, "p05": 0.05758974328637123, "p95": 0.10452556610107422 }, { "label": "completed_group_5", "mean": 0.08982288837432861, "median": 0.08912864327430725, "p05": 0.0663854256272316, "p95": 0.11604885756969452 }, { "label": "current_group_partial", "mean": 0.5543455481529236, "median": 0.5596089363098145, "p05": 0.45455610752105713, "p95": 0.6367484331130981 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.7241814136505127, "group": 6, "latest_source_mass": 0.39484071731567383, "layer": 22, "max_source_mass": 0.39484071731567383, "mixer_index": 42, "normalized_entropy": 0.886053970419732, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.13360010087490082, "median": 0.14072942733764648, "p05": 0.07699814438819885, "p95": 0.17506597936153412 }, { "label": "completed_group_1", "mean": 0.10609637200832367, "median": 0.10586297512054443, "p05": 0.08648421615362167, "p95": 0.12602579593658447 }, { "label": "completed_group_2", "mean": 0.06363911926746368, "median": 0.06324908882379532, "p05": 0.054980404675006866, "p95": 0.07316673547029495 }, { "label": "completed_group_3", "mean": 0.07426095008850098, "median": 0.07362888753414154, "p05": 0.06318790465593338, "p95": 0.08665015548467636 }, { "label": "completed_group_4", "mean": 0.09383076429367065, "median": 0.09294568747282028, "p05": 0.07638302445411682, "p95": 0.11445100605487823 }, { "label": "completed_group_5", "mean": 0.13373197615146637, "median": 0.13327741622924805, "p05": 0.11399447172880173, "p95": 0.15641780197620392 }, { "label": "current_group_partial", "mean": 0.39484071731567383, "median": 0.3879151940345764, "p05": 0.3302172124385834, "p95": 0.4852263331413269 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.3979300260543823, "group": 6, "latest_source_mass": 0.5830056071281433, "layer": 22, "max_source_mass": 0.5830056071281433, "mixer_index": 43, "normalized_entropy": 0.7183939231382495, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.08959861099720001, "median": 0.07544538378715515, "p05": 0.04165133833885193, "p95": 0.20099139213562012 }, { "label": "completed_group_1", "mean": 0.05414731800556183, "median": 0.05336771905422211, "p05": 0.04069708660244942, "p95": 0.07046990841627121 }, { "label": "completed_group_2", "mean": 0.07668881118297577, "median": 0.07607124000787735, "p05": 0.0555989071726799, "p95": 0.09980960935354233 }, { "label": "completed_group_3", "mean": 0.05603286623954773, "median": 0.054573021829128265, "p05": 0.04162490367889404, "p95": 0.0730118378996849 }, { "label": "completed_group_4", "mean": 0.06979228556156158, "median": 0.06884932518005371, "p05": 0.04576495662331581, "p95": 0.09806846082210541 }, { "label": "completed_group_5", "mean": 0.07073447108268738, "median": 0.07023533433675766, "p05": 0.04956750199198723, "p95": 0.09379896521568298 }, { "label": "current_group_partial", "mean": 0.5830056071281433, "median": 0.5862616896629333, "p05": 0.4658474922180176, "p95": 0.6849814057350159 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.7459560632705688, "group": 6, "latest_source_mass": 0.37691548466682434, "layer": 23, "max_source_mass": 0.37691548466682434, "mixer_index": 44, "normalized_entropy": 0.897243926765161, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.14318783581256866, "median": 0.1493408977985382, "p05": 0.08292607218027115, "p95": 0.19464610517024994 }, { "label": "completed_group_1", "mean": 0.1094653457403183, "median": 0.10858799517154694, "p05": 0.09432064741849899, "p95": 0.1279931515455246 }, { "label": "completed_group_2", "mean": 0.06349228322505951, "median": 0.06325754523277283, "p05": 0.05439673364162445, "p95": 0.0737837627530098 }, { "label": "completed_group_3", "mean": 0.07859592139720917, "median": 0.0775163471698761, "p05": 0.06603655964136124, "p95": 0.09357087314128876 }, { "label": "completed_group_4", "mean": 0.09567482024431229, "median": 0.09468592703342438, "p05": 0.07671141624450684, "p95": 0.11774884164333344 }, { "label": "completed_group_5", "mean": 0.13266831636428833, "median": 0.13199329376220703, "p05": 0.11244359612464905, "p95": 0.15504513680934906 }, { "label": "current_group_partial", "mean": 0.37691548466682434, "median": 0.37180638313293457, "p05": 0.31207239627838135, "p95": 0.4595233201980591 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.4055393934249878, "group": 6, "latest_source_mass": 0.5772827863693237, "layer": 23, "max_source_mass": 0.5772827863693237, "mixer_index": 45, "normalized_entropy": 0.7223043644164862, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.10154955089092255, "median": 0.08766824007034302, "p05": 0.04901552200317383, "p95": 0.20287738740444183 }, { "label": "completed_group_1", "mean": 0.05619276314973831, "median": 0.055477917194366455, "p05": 0.041608311235904694, "p95": 0.07371112704277039 }, { "label": "completed_group_2", "mean": 0.07791627198457718, "median": 0.07736755907535553, "p05": 0.055920060724020004, "p95": 0.10158214718103409 }, { "label": "completed_group_3", "mean": 0.053915880620479584, "median": 0.05215863138437271, "p05": 0.03798118978738785, "p95": 0.07156119495630264 }, { "label": "completed_group_4", "mean": 0.06398355215787888, "median": 0.06229078769683838, "p05": 0.040616974234580994, "p95": 0.09270931780338287 }, { "label": "completed_group_5", "mean": 0.06915920227766037, "median": 0.0684593990445137, "p05": 0.046986356377601624, "p95": 0.09408729523420334 }, { "label": "current_group_partial", "mean": 0.5772827863693237, "median": 0.5824689269065857, "p05": 0.4518115520477295, "p95": 0.6926202178001404 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.7853620052337646, "group": 6, "latest_source_mass": 0.3407441973686218, "layer": 24, "max_source_mass": 0.3407441973686218, "mixer_index": 46, "normalized_entropy": 0.9174945750195659, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.15284113585948944, "median": 0.1593906283378601, "p05": 0.08678612112998962, "p95": 0.211342453956604 }, { "label": "completed_group_1", "mean": 0.12206703424453735, "median": 0.12153112143278122, "p05": 0.1060115247964859, "p95": 0.1399766057729721 }, { "label": "completed_group_2", "mean": 0.06508444249629974, "median": 0.06482555717229843, "p05": 0.055350616574287415, "p95": 0.07557765394449234 }, { "label": "completed_group_3", "mean": 0.08347313106060028, "median": 0.08190013468265533, "p05": 0.06966532766819, "p95": 0.10050469636917114 }, { "label": "completed_group_4", "mean": 0.09811970591545105, "median": 0.09664031863212585, "p05": 0.07688035070896149, "p95": 0.12369741499423981 }, { "label": "completed_group_5", "mean": 0.13767032325267792, "median": 0.1373325139284134, "p05": 0.11642646044492722, "p95": 0.1606694608926773 }, { "label": "current_group_partial", "mean": 0.3407441973686218, "median": 0.33805108070373535, "p05": 0.271301805973053, "p95": 0.41781002283096313 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.509842872619629, "group": 6, "latest_source_mass": 0.5227814316749573, "layer": 24, "max_source_mass": 0.5227814316749573, "mixer_index": 47, "normalized_entropy": 0.7759057494780099, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.11788781732320786, "median": 0.10097194463014603, "p05": 0.06265684217214584, "p95": 0.2408325970172882 }, { "label": "completed_group_1", "mean": 0.06822571158409119, "median": 0.0673636943101883, "p05": 0.05209728330373764, "p95": 0.08648915588855743 }, { "label": "completed_group_2", "mean": 0.08556114137172699, "median": 0.08463940769433975, "p05": 0.06234566867351532, "p95": 0.1125786080956459 }, { "label": "completed_group_3", "mean": 0.06067090481519699, "median": 0.058974385261535645, "p05": 0.043267179280519485, "p95": 0.07997901737689972 }, { "label": "completed_group_4", "mean": 0.07228855788707733, "median": 0.06972275674343109, "p05": 0.04689513146877289, "p95": 0.106512151658535 }, { "label": "completed_group_5", "mean": 0.07258446514606476, "median": 0.07082539796829224, "p05": 0.050949547439813614, "p95": 0.09970614314079285 }, { "label": "current_group_partial", "mean": 0.5227814316749573, "median": 0.5290273427963257, "p05": 0.38773614168167114, "p95": 0.6392589807510376 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.839367389678955, "group": 7, "latest_source_mass": 0.29583215713500977, "layer": 25, "max_source_mass": 0.29583215713500977, "mixer_index": 48, "normalized_entropy": 0.9452478525649903, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.15906387567520142, "median": 0.15846745669841766, "p05": 0.09519976377487183, "p95": 0.24517810344696045 }, { "label": "completed_group_1", "mean": 0.13139380514621735, "median": 0.1308426856994629, "p05": 0.11478274315595627, "p95": 0.15000829100608826 }, { "label": "completed_group_2", "mean": 0.07717745751142502, "median": 0.07715500891208649, "p05": 0.06616044044494629, "p95": 0.08791667222976685 }, { "label": "completed_group_3", "mean": 0.09607303142547607, "median": 0.09436552226543427, "p05": 0.08152313530445099, "p95": 0.11384639143943787 }, { "label": "completed_group_4", "mean": 0.1107850894331932, "median": 0.1089654266834259, "p05": 0.08814431726932526, "p95": 0.139452263712883 }, { "label": "completed_group_5", "mean": 0.12967455387115479, "median": 0.1299237608909607, "p05": 0.10832332074642181, "p95": 0.15138982236385345 }, { "label": "completed_group_6", "mean": 0.29583215713500977, "median": 0.29647016525268555, "p05": 0.2276897132396698, "p95": 0.3604946434497833 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.5841834545135498, "group": 7, "latest_source_mass": 0.40767741203308105, "layer": 25, "max_source_mass": 0.40767741203308105, "mixer_index": 49, "normalized_entropy": 0.7618312045616817, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.07896879315376282, "median": 0.06809917092323303, "p05": 0.025932887569069862, "p95": 0.19368338584899902 }, { "label": "completed_group_1", "mean": 0.048752427101135254, "median": 0.04767637699842453, "p05": 0.034443892538547516, "p95": 0.0667153000831604 }, { "label": "completed_group_2", "mean": 0.055707164108753204, "median": 0.0549345538020134, "p05": 0.038696639239788055, "p95": 0.0750277042388916 }, { "label": "completed_group_3", "mean": 0.04139826446771622, "median": 0.03878612071275711, "p05": 0.026673054322600365, "p95": 0.05746251344680786 }, { "label": "completed_group_4", "mean": 0.04449018836021423, "median": 0.04196460545063019, "p05": 0.026162121444940567, "p95": 0.07164917886257172 }, { "label": "completed_group_5", "mean": 0.04285799711942673, "median": 0.04109475016593933, "p05": 0.029393533244729042, "p95": 0.06277710199356079 }, { "label": "completed_group_6", "mean": 0.2801477909088135, "median": 0.2763082981109619, "p05": 0.1455708146095276, "p95": 0.4254714846611023 }, { "label": "current_group_partial", "mean": 0.40767741203308105, "median": 0.4120897948741913, "p05": 0.21399420499801636, "p95": 0.5853031277656555 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.8125314712524414, "group": 7, "latest_source_mass": 0.3700707256793976, "layer": 26, "max_source_mass": 0.3700707256793976, "mixer_index": 50, "normalized_entropy": 0.8716433883436914, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.10889513790607452, "median": 0.11333495378494263, "p05": 0.04850132763385773, "p95": 0.1620434820652008 }, { "label": "completed_group_1", "mean": 0.08656036853790283, "median": 0.08640051633119583, "p05": 0.0650845319032669, "p95": 0.1084962785243988 }, { "label": "completed_group_2", "mean": 0.048123255372047424, "median": 0.04800213500857353, "p05": 0.03576180338859558, "p95": 0.06107594817876816 }, { "label": "completed_group_3", "mean": 0.06561170518398285, "median": 0.06379580497741699, "p05": 0.050968848168849945, "p95": 0.08332888036966324 }, { "label": "completed_group_4", "mean": 0.06917259097099304, "median": 0.06724178791046143, "p05": 0.050207968801259995, "p95": 0.09373342245817184 }, { "label": "completed_group_5", "mean": 0.08286689221858978, "median": 0.08183546364307404, "p05": 0.06535022705793381, "p95": 0.10340933501720428 }, { "label": "completed_group_6", "mean": 0.16869932413101196, "median": 0.16509854793548584, "p05": 0.11059704422950745, "p95": 0.23654375970363617 }, { "label": "current_group_partial", "mean": 0.3700707256793976, "median": 0.3694133758544922, "p05": 0.23441238701343536, "p95": 0.5025027990341187 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.50089693069458, "group": 7, "latest_source_mass": 0.49096518754959106, "layer": 26, "max_source_mass": 0.49096518754959106, "mixer_index": 51, "normalized_entropy": 0.7217788529328457, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.07974998652935028, "median": 0.07402504235506058, "p05": 0.04579866677522659, "p95": 0.1359037309885025 }, { "label": "completed_group_1", "mean": 0.04166398197412491, "median": 0.04038450866937637, "p05": 0.027992289513349533, "p95": 0.05969971418380737 }, { "label": "completed_group_2", "mean": 0.04607272148132324, "median": 0.044438350945711136, "p05": 0.030038591474294662, "p95": 0.06808309257030487 }, { "label": "completed_group_3", "mean": 0.03586764633655548, "median": 0.03317485377192497, "p05": 0.02203454077243805, "p95": 0.05572832003235817 }, { "label": "completed_group_4", "mean": 0.04223340004682541, "median": 0.03930673003196716, "p05": 0.02465122565627098, "p95": 0.06968094408512115 }, { "label": "completed_group_5", "mean": 0.04095568507909775, "median": 0.03919431194663048, "p05": 0.026239857077598572, "p95": 0.06058088690042496 }, { "label": "completed_group_6", "mean": 0.22249138355255127, "median": 0.2125401496887207, "p05": 0.10647299885749817, "p95": 0.3711881935596466 }, { "label": "current_group_partial", "mean": 0.49096518754959106, "median": 0.4988659620285034, "p05": 0.2878011167049408, "p95": 0.6676807403564453 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.8059929609298706, "group": 7, "latest_source_mass": 0.37139976024627686, "layer": 27, "max_source_mass": 0.37139976024627686, "mixer_index": 52, "normalized_entropy": 0.8684990295379666, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.10711923241615295, "median": 0.10972198843955994, "p05": 0.04896187782287598, "p95": 0.16201043128967285 }, { "label": "completed_group_1", "mean": 0.07651707530021667, "median": 0.07666032761335373, "p05": 0.05141128972172737, "p95": 0.10152198374271393 }, { "label": "completed_group_2", "mean": 0.058284737169742584, "median": 0.05812416970729828, "p05": 0.039273619651794434, "p95": 0.07745087146759033 }, { "label": "completed_group_3", "mean": 0.06609587371349335, "median": 0.06383530795574188, "p05": 0.04481930285692215, "p95": 0.09336265176534653 }, { "label": "completed_group_4", "mean": 0.07139429450035095, "median": 0.0694514587521553, "p05": 0.0485839769244194, "p95": 0.10034126043319702 }, { "label": "completed_group_5", "mean": 0.07493151724338531, "median": 0.07319007813930511, "p05": 0.051876723766326904, "p95": 0.10402432829141617 }, { "label": "completed_group_6", "mean": 0.17425751686096191, "median": 0.16937679052352905, "p05": 0.10449051856994629, "p95": 0.2624811828136444 }, { "label": "current_group_partial", "mean": 0.37139976024627686, "median": 0.3732905089855194, "p05": 0.23830299079418182, "p95": 0.5014542937278748 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.5622732639312744, "group": 7, "latest_source_mass": 0.4768889844417572, "layer": 27, "max_source_mass": 0.4768889844417572, "mixer_index": 53, "normalized_entropy": 0.7512946301290215, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.08775827288627625, "median": 0.08504067361354828, "p05": 0.0428270623087883, "p95": 0.144285649061203 }, { "label": "completed_group_1", "mean": 0.047765426337718964, "median": 0.04646281152963638, "p05": 0.031814731657505035, "p95": 0.06807165592908859 }, { "label": "completed_group_2", "mean": 0.053928617388010025, "median": 0.052262187004089355, "p05": 0.035437777638435364, "p95": 0.0784272700548172 }, { "label": "completed_group_3", "mean": 0.04293380305171013, "median": 0.03996016085147858, "p05": 0.02556057646870613, "p95": 0.06623870134353638 }, { "label": "completed_group_4", "mean": 0.04174989461898804, "median": 0.03903091698884964, "p05": 0.025013400241732597, "p95": 0.06869364529848099 }, { "label": "completed_group_5", "mean": 0.04734005779027939, "median": 0.04609068110585213, "p05": 0.029884062707424164, "p95": 0.06911227852106094 }, { "label": "completed_group_6", "mean": 0.20163494348526, "median": 0.19222964346408844, "p05": 0.09963390231132507, "p95": 0.3307305574417114 }, { "label": "current_group_partial", "mean": 0.4768889844417572, "median": 0.48383551836013794, "p05": 0.276212215423584, "p95": 0.6487575173377991 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.8757455348968506, "group": 7, "latest_source_mass": 0.3109493851661682, "layer": 28, "max_source_mass": 0.3109493851661682, "mixer_index": 54, "normalized_entropy": 0.9020429270551009, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.11563669890165329, "median": 0.12087491154670715, "p05": 0.05733206868171692, "p95": 0.1688953936100006 }, { "label": "completed_group_1", "mean": 0.08108697831630707, "median": 0.08185897022485733, "p05": 0.05175650864839554, "p95": 0.10753051191568375 }, { "label": "completed_group_2", "mean": 0.07280858606100082, "median": 0.07357072830200195, "p05": 0.04870978742837906, "p95": 0.0948689877986908 }, { "label": "completed_group_3", "mean": 0.07553227245807648, "median": 0.07362619787454605, "p05": 0.056317005306482315, "p95": 0.09756095707416534 }, { "label": "completed_group_4", "mean": 0.07606014609336853, "median": 0.07456053793430328, "p05": 0.05528895929455757, "p95": 0.1022825762629509 }, { "label": "completed_group_5", "mean": 0.06935927271842957, "median": 0.06843917071819305, "p05": 0.04978285729885101, "p95": 0.09167985618114471 }, { "label": "completed_group_6", "mean": 0.19856664538383484, "median": 0.19508783519268036, "p05": 0.1281973123550415, "p95": 0.27796900272369385 }, { "label": "current_group_partial", "mean": 0.3109493851661682, "median": 0.3105033040046692, "p05": 0.18172834813594818, "p95": 0.43916669487953186 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.6924852132797241, "group": 7, "latest_source_mass": 0.3315581679344177, "layer": 28, "max_source_mass": 0.3315581679344177, "mixer_index": 55, "normalized_entropy": 0.8139133413255193, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.08638577908277512, "median": 0.058608733117580414, "p05": 0.026544082909822464, "p95": 0.2740764319896698 }, { "label": "completed_group_1", "mean": 0.059912845492362976, "median": 0.05901291221380234, "p05": 0.03629481792449951, "p95": 0.08707985281944275 }, { "label": "completed_group_2", "mean": 0.06609142571687698, "median": 0.0647939145565033, "p05": 0.038691695779561996, "p95": 0.09703366458415985 }, { "label": "completed_group_3", "mean": 0.056745611131191254, "median": 0.05418796092271805, "p05": 0.032174739986658096, "p95": 0.08370660990476608 }, { "label": "completed_group_4", "mean": 0.06277773529291153, "median": 0.0600711889564991, "p05": 0.03501531109213829, "p95": 0.10094858705997467 }, { "label": "completed_group_5", "mean": 0.05352145433425903, "median": 0.05179818347096443, "p05": 0.032271772623062134, "p95": 0.08084158599376678 }, { "label": "completed_group_6", "mean": 0.2830069661140442, "median": 0.2759752869606018, "p05": 0.12981083989143372, "p95": 0.46204638481140137 }, { "label": "current_group_partial", "mean": 0.3315581679344177, "median": 0.3204721510410309, "p05": 0.13446375727653503, "p95": 0.5667697787284851 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.8980522155761719, "group": 8, "latest_source_mass": 0.20123262703418732, "layer": 29, "max_source_mass": 0.26602888107299805, "mixer_index": 56, "normalized_entropy": 0.9127701729200177, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.11603894829750061, "median": 0.09991352260112762, "p05": 0.030035395175218582, "p95": 0.2941743731498718 }, { "label": "completed_group_1", "mean": 0.08804173022508621, "median": 0.08880250155925751, "p05": 0.05664174258708954, "p95": 0.11779069900512695 }, { "label": "completed_group_2", "mean": 0.07975024729967117, "median": 0.07984748482704163, "p05": 0.05428915470838547, "p95": 0.10499341040849686 }, { "label": "completed_group_3", "mean": 0.09027000516653061, "median": 0.08898856490850449, "p05": 0.06189651042222977, "p95": 0.11996553838253021 }, { "label": "completed_group_4", "mean": 0.08430293202400208, "median": 0.0819794088602066, "p05": 0.05464346706867218, "p95": 0.12159879505634308 }, { "label": "completed_group_5", "mean": 0.07433462142944336, "median": 0.07287181913852692, "p05": 0.052520088851451874, "p95": 0.1010315865278244 }, { "label": "completed_group_6", "mean": 0.26602888107299805, "median": 0.2639757990837097, "p05": 0.15474572777748108, "p95": 0.3846370279788971 }, { "label": "completed_group_7", "mean": 0.20123262703418732, "median": 0.18140269815921783, "p05": 0.08419936150312424, "p95": 0.3782624900341034 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.7512003183364868, "group": 8, "latest_source_mass": 0.33482813835144043, "layer": 29, "max_source_mass": 0.33482813835144043, "mixer_index": 57, "normalized_entropy": 0.7970056117156376, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.07744461297988892, "median": 0.06970229744911194, "p05": 0.014401955530047417, "p95": 0.17144526541233063 }, { "label": "completed_group_1", "mean": 0.046086348593235016, "median": 0.04394831880927086, "p05": 0.028066720813512802, "p95": 0.0718703418970108 }, { "label": "completed_group_2", "mean": 0.05025433003902435, "median": 0.04861236736178398, "p05": 0.029644077643752098, "p95": 0.07761462777853012 }, { "label": "completed_group_3", "mean": 0.03748488053679466, "median": 0.03434944897890091, "p05": 0.020774798467755318, "p95": 0.05938909202814102 }, { "label": "completed_group_4", "mean": 0.04005061462521553, "median": 0.037498839199543, "p05": 0.019541850313544273, "p95": 0.06922487914562225 }, { "label": "completed_group_5", "mean": 0.041623204946517944, "median": 0.039552345871925354, "p05": 0.022081367671489716, "p95": 0.06821337342262268 }, { "label": "completed_group_6", "mean": 0.198423832654953, "median": 0.18318486213684082, "p05": 0.07548617571592331, "p95": 0.3721975088119507 }, { "label": "completed_group_7", "mean": 0.17380405962467194, "median": 0.1514919549226761, "p05": 0.05777408182621002, "p95": 0.36326974630355835 }, { "label": "current_group_partial", "mean": 0.33482813835144043, "median": 0.33602219820022583, "p05": 0.0795319601893425, "p95": 0.5979493856430054 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.0160694122314453, "group": 8, "latest_source_mass": 0.21711516380310059, "layer": 30, "max_source_mass": 0.21711516380310059, "mixer_index": 58, "normalized_entropy": 0.9175527313077866, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.09385271370410919, "median": 0.09211117774248123, "p05": 0.032377734780311584, "p95": 0.16496515274047852 }, { "label": "completed_group_1", "mean": 0.06208505108952522, "median": 0.06140740588307381, "p05": 0.0407925546169281, "p95": 0.0863042101264 }, { "label": "completed_group_2", "mean": 0.06560768187046051, "median": 0.06516212224960327, "p05": 0.041897669434547424, "p95": 0.09085705131292343 }, { "label": "completed_group_3", "mean": 0.07029052078723907, "median": 0.06780634075403214, "p05": 0.04846132919192314, "p95": 0.09917552024126053 }, { "label": "completed_group_4", "mean": 0.06935097277164459, "median": 0.06708167493343353, "p05": 0.04668639227747917, "p95": 0.09815650433301926 }, { "label": "completed_group_5", "mean": 0.056832700967788696, "median": 0.05595892667770386, "p05": 0.038822539150714874, "p95": 0.0771557092666626 }, { "label": "completed_group_6", "mean": 0.19425815343856812, "median": 0.19145764410495758, "p05": 0.12249615788459778, "p95": 0.27496030926704407 }, { "label": "completed_group_7", "mean": 0.17060703039169312, "median": 0.16317546367645264, "p05": 0.09239593893289566, "p95": 0.2726203501224518 }, { "label": "current_group_partial", "mean": 0.21711516380310059, "median": 0.21198657155036926, "p05": 0.11019522696733475, "p95": 0.3391069769859314 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 1.7837426662445068, "group": 8, "latest_source_mass": 0.29026859998703003, "layer": 30, "max_source_mass": 0.29026859998703003, "mixer_index": 59, "normalized_entropy": 0.8118162725118464, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.08836327493190765, "median": 0.07904411852359772, "p05": 0.02631576545536518, "p95": 0.17534467577934265 }, { "label": "completed_group_1", "mean": 0.04953558370471001, "median": 0.04658038914203644, "p05": 0.025181813165545464, "p95": 0.08318150043487549 }, { "label": "completed_group_2", "mean": 0.04813014343380928, "median": 0.045740917325019836, "p05": 0.026714935898780823, "p95": 0.07843323051929474 }, { "label": "completed_group_3", "mean": 0.0373263955116272, "median": 0.03401809185743332, "p05": 0.01990434154868126, "p95": 0.06290259957313538 }, { "label": "completed_group_4", "mean": 0.037015400826931, "median": 0.03405017405748367, "p05": 0.018646355718374252, "p95": 0.06544196605682373 }, { "label": "completed_group_5", "mean": 0.03390160948038101, "median": 0.03198609501123428, "p05": 0.018189728260040283, "p95": 0.05570102855563164 }, { "label": "completed_group_6", "mean": 0.1840541511774063, "median": 0.17327937483787537, "p05": 0.07966600358486176, "p95": 0.32982927560806274 }, { "label": "completed_group_7", "mean": 0.23140482604503632, "median": 0.22406929731369019, "p05": 0.08831030130386353, "p95": 0.40350067615509033 }, { "label": "current_group_partial", "mean": 0.29026859998703003, "median": 0.2745037376880646, "p05": 0.10280418395996094, "p95": 0.5399085879325867 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 1.9973840713500977, "group": 8, "latest_source_mass": 0.19101998209953308, "layer": 31, "max_source_mass": 0.20831164717674255, "mixer_index": 60, "normalized_entropy": 0.9090486661912383, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.10261696577072144, "median": 0.10389091819524765, "p05": 0.01371697522699833, "p95": 0.21163305640220642 }, { "label": "completed_group_1", "mean": 0.06039813905954361, "median": 0.06085003539919853, "p05": 0.026107093319296837, "p95": 0.09348798543214798 }, { "label": "completed_group_2", "mean": 0.06671924144029617, "median": 0.06656201928853989, "p05": 0.034684643149375916, "p95": 0.09819839894771576 }, { "label": "completed_group_3", "mean": 0.08377155661582947, "median": 0.0804532840847969, "p05": 0.05448959022760391, "p95": 0.12039553374052048 }, { "label": "completed_group_4", "mean": 0.07002593576908112, "median": 0.06737451255321503, "p05": 0.044841229915618896, "p95": 0.10655929893255234 }, { "label": "completed_group_5", "mean": 0.05768897384405136, "median": 0.056077782064676285, "p05": 0.039495617151260376, "p95": 0.08242568373680115 }, { "label": "completed_group_6", "mean": 0.20831164717674255, "median": 0.2021285891532898, "p05": 0.11652229726314545, "p95": 0.31706833839416504 }, { "label": "completed_group_7", "mean": 0.1594475358724594, "median": 0.1546279788017273, "p05": 0.06593266129493713, "p95": 0.2735573351383209 }, { "label": "current_group_partial", "mean": 0.19101998209953308, "median": 0.1713225245475769, "p05": 0.06408411264419556, "p95": 0.3850628137588501 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 1.969987154006958, "group": 8, "latest_source_mass": 0.16864418983459473, "layer": 31, "max_source_mass": 0.24503237009048462, "mixer_index": 61, "normalized_entropy": 0.8965797917640489, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.08120635151863098, "median": 0.061164144426584244, "p05": 0.0259944386780262, "p95": 0.23622018098831177 }, { "label": "completed_group_1", "mean": 0.07894632965326309, "median": 0.07658770680427551, "p05": 0.0496862456202507, "p95": 0.11596276611089706 }, { "label": "completed_group_2", "mean": 0.08615730702877045, "median": 0.08467075228691101, "p05": 0.050409119576215744, "p95": 0.12811316549777985 }, { "label": "completed_group_3", "mean": 0.07609779387712479, "median": 0.0722871869802475, "p05": 0.04099677503108978, "p95": 0.11950768530368805 }, { "label": "completed_group_4", "mean": 0.058728985488414764, "median": 0.056080836802721024, "p05": 0.03323862701654434, "p95": 0.09344407916069031 }, { "label": "completed_group_5", "mean": 0.05076971277594566, "median": 0.04946017265319824, "p05": 0.03047419898211956, "p95": 0.07563400268554688 }, { "label": "completed_group_6", "mean": 0.24503237009048462, "median": 0.24110376834869385, "p05": 0.10613486170768738, "p95": 0.3991503119468689 }, { "label": "completed_group_7", "mean": 0.1544169783592224, "median": 0.1400495171546936, "p05": 0.06199025735259056, "p95": 0.2995939254760742 }, { "label": "current_group_partial", "mean": 0.16864418983459473, "median": 0.15222924947738647, "p05": 0.0543956533074379, "p95": 0.3329208791255951 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.071648359298706, "group": 8, "latest_source_mass": 0.11520501971244812, "layer": 32, "max_source_mass": 0.20955611765384674, "mixer_index": 62, "normalized_entropy": 0.9428478002054053, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.10569609701633453, "median": 0.08689188957214355, "p05": 0.03173588588833809, "p95": 0.24571751058101654 }, { "label": "completed_group_1", "mean": 0.09922641515731812, "median": 0.09795533865690231, "p05": 0.06407676637172699, "p95": 0.14120742678642273 }, { "label": "completed_group_2", "mean": 0.08180510997772217, "median": 0.080217644572258, "p05": 0.054607540369033813, "p95": 0.11442367732524872 }, { "label": "completed_group_3", "mean": 0.10646799206733704, "median": 0.10510502755641937, "p05": 0.07415887713432312, "p95": 0.14216329157352448 }, { "label": "completed_group_4", "mean": 0.0826798751950264, "median": 0.08022409677505493, "p05": 0.05491747707128525, "p95": 0.11860857903957367 }, { "label": "completed_group_5", "mean": 0.06615597754716873, "median": 0.06485182791948318, "p05": 0.04477996751666069, "p95": 0.09238749742507935 }, { "label": "completed_group_6", "mean": 0.20955611765384674, "median": 0.206287682056427, "p05": 0.129981130361557, "p95": 0.2996031939983368 }, { "label": "completed_group_7", "mean": 0.13320739567279816, "median": 0.12465103715658188, "p05": 0.06303662061691284, "p95": 0.23065781593322754 }, { "label": "current_group_partial", "mean": 0.11520501971244812, "median": 0.10386130958795547, "p05": 0.03912060335278511, "p95": 0.22735844552516937 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 1.9781701564788818, "group": 8, "latest_source_mass": 0.16112206876277924, "layer": 32, "max_source_mass": 0.22251944243907928, "mixer_index": 63, "normalized_entropy": 0.9003040366848136, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.09446848928928375, "median": 0.074150450527668, "p05": 0.042592622339725494, "p95": 0.19443637132644653 }, { "label": "completed_group_1", "mean": 0.07056106626987457, "median": 0.06868506222963333, "p05": 0.04818308353424072, "p95": 0.0982733890414238 }, { "label": "completed_group_2", "mean": 0.08033448457717896, "median": 0.08047192543745041, "p05": 0.048296794295310974, "p95": 0.11253741383552551 }, { "label": "completed_group_3", "mean": 0.0658978745341301, "median": 0.06256154924631119, "p05": 0.04223886504769325, "p95": 0.0967884361743927 }, { "label": "completed_group_4", "mean": 0.058234501630067825, "median": 0.0563921295106411, "p05": 0.03529864549636841, "p95": 0.08846980333328247 }, { "label": "completed_group_5", "mean": 0.04316519945859909, "median": 0.042268507182598114, "p05": 0.029443470761179924, "p95": 0.05996086448431015 }, { "label": "completed_group_6", "mean": 0.2036968618631363, "median": 0.19645483791828156, "p05": 0.09398645162582397, "p95": 0.3395216464996338 }, { "label": "completed_group_7", "mean": 0.22251944243907928, "median": 0.2130982130765915, "p05": 0.11867167800664902, "p95": 0.35465875267982483 }, { "label": "current_group_partial", "mean": 0.16112206876277924, "median": 0.14023703336715698, "p05": 0.06008770316839218, "p95": 0.3247341811656952 } ], "sources": 9 }, { "branch": "output", "entropy_mean": 1.6036605834960938, "group": null, "latest_source_mass": 0.31639498472213745, "layer": null, "max_source_mass": 0.3412485122680664, "mixer_index": null, "normalized_entropy": 0.7298573846467136, "offset": null, "source_summaries": [ { "label": "embedding", "mean": 0.09759747982025146, "median": 0.09915056824684143, "p05": 0.025461740791797638, "p95": 0.16398483514785767 }, { "label": "completed_group_1", "mean": 0.04215940088033676, "median": 0.04050712287425995, "p05": 0.025720715522766113, "p95": 0.06398092210292816 }, { "label": "completed_group_2", "mean": 0.041145481169223785, "median": 0.03881259262561798, "p05": 0.023510465398430824, "p95": 0.0662468671798706 }, { "label": "completed_group_3", "mean": 0.029554028064012527, "median": 0.027420252561569214, "p05": 0.01627083122730255, "p95": 0.04924973472952843 }, { "label": "completed_group_4", "mean": 0.0289996936917305, "median": 0.027226522564888, "p05": 0.01560397818684578, "p95": 0.048351090401411057 }, { "label": "completed_group_5", "mean": 0.03647579252719879, "median": 0.0347352996468544, "p05": 0.020744025707244873, "p95": 0.058257561177015305 }, { "label": "completed_group_6", "mean": 0.06642461568117142, "median": 0.057809896767139435, "p05": 0.022837381809949875, "p95": 0.14078694581985474 }, { "label": "completed_group_7", "mean": 0.3412485122680664, "median": 0.3336990475654602, "p05": 0.0819513276219368, "p95": 0.6423859596252441 }, { "label": "completed_group_8", "mean": 0.31639498472213745, "median": 0.30713334679603577, "p05": 0.05841828137636185, "p95": 0.6156357526779175 } ], "sources": 9 } ], "mode": "uniform_all", "optimizer_state_after": "ed03e6fbd4a12d8b063dcb22e0437754285f54d585374cd52fbd534f05d24637", "optimizer_state_before": "ed03e6fbd4a12d8b063dcb22e0437754285f54d585374cd52fbd534f05d24637", "optimizer_state_unchanged": true, "positions": { "attention_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "5e6ecbcd78543fc42d4098a51169dca446d404ce384015643e84d2c78433c949", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.0933690071105957, 0.24537226557731628, 0.07698935270309448, 0.0523640513420105, 0.09380833059549332, 0.03647654876112938, 0.039120711386203766, 0.06371714919805527, 0.05156813934445381, 0.03728010877966881, 0.03983496502041817, 0.040284499526023865, 0.046871740370988846, 0.04314620420336723, 0.04033593460917473, 0.04471700266003609, 0.08397205173969269, 0.0528092160820961, 0.04443390294909477, 0.07560808956623077, 0.03708760067820549, 0.03599191829562187, 0.033304885029792786, 0.04061548039317131, 0.025487789884209633, 0.021565141156315804, 0.02342291921377182, 0.017244853079319, 0.024166326969861984, 0.016819356009364128, 0.021590016782283783, 0.026201345026493073 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 8.459922923975682e-06, "first_to_last_ratio": 2.328407895763024, "imbalance_abs_log_ratio": 0.8451847274778195, "last_quartile_mean": 3.6333509001451603e-06, "mean": 5.48445950698806e-06, "normalized": [ 1.7981492239015215, 1.9357513407040896, 1.6672073870519302, 1.5017840953582675, 1.4824821685817133, 1.4014170610824435, 1.3280238040790173, 1.2253951633993214, 1.156695262052324, 1.0641431906294738, 1.0106344255365511, 0.9662588157958233, 0.9475559543612537, 0.9090700068173221, 0.8837794995113782, 0.8574858790556309, 0.8607721566700557, 0.837358195636748, 0.8360590737297285, 0.824566471544106, 0.8217232949937042, 0.8060327579708028, 0.7938760845984737, 0.7839296928623085, 0.7744755711517991, 0.7571739836659549, 0.7307992717355416, 0.68650330244616, 0.6478013597166409, 0.6078235943688355, 0.5594006660114121, 0.5358712449796666 ], "peak_layer": 2, "peak_normalized": 1.9357513407040896, "population_cv": 0.35649081874965816, "rest_mean": 5.691642602161866e-06, "spike_contrast": 0.7670317864637676, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.365670793049503e-06, "top_five_layers": [ 2, 1, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 9.8618766060099e-06, 1.0616549843689427e-05, 9.14373140403768e-06, 8.236474059231114e-06, 8.130613423418254e-06, 7.686015123908874e-06, 7.283492777787615e-06, 6.720630153722595e-06, 6.343848326650914e-06, 5.836250238644425e-06, 5.542783583223354e-06, 5.299407348502427e-06, 5.196832262299722e-06, 4.985757641406963e-06, 4.847052878176328e-06, 4.7028465814946685e-06, 4.720870037999703e-06, 4.59245711681433e-06, 4.585332135320641e-06, 4.522301424003672e-06, 4.506708137341775e-06, 4.420654022396775e-06, 4.353981239546556e-06, 4.2994306568289176e-06, 4.247579909133492e-06, 4.152690053160768e-06, 4.008039013569942e-06, 3.7650995636795415e-06, 3.552840325937723e-06, 3.333583890707814e-06, 3.068010300921742e-06, 2.93896414405026e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 3.357288642291678e-05, "first_to_last_ratio": 2.3058080996835404, "imbalance_abs_log_ratio": 0.8354312005538636, "last_quartile_mean": 1.456013899314712e-05, "mean": 2.1838724165945678e-05, "normalized": [ 1.8092622254038118, 1.9431844817635626, 1.6767107875669782, 1.4950940749576347, 1.4674045069810522, 1.3807616632021802, 1.3135886403618173, 1.2124726583027057, 1.1458806799401573, 1.058561922179424, 1.0064538267556071, 0.9652676350426349, 0.9479497443118112, 0.9127224235896364, 0.8880734273823512, 0.8621668896719061, 0.8678176638273486, 0.8415578850574983, 0.8383782990632195, 0.8250966517885899, 0.8217974585881554, 0.8072470254003167, 0.7943189523527976, 0.7845346497712516, 0.7759095209279144, 0.7590478200773931, 0.7335564093275885, 0.6907997066984136, 0.653253330518448, 0.6137700422385202, 0.5655418513252468, 0.5418171456240263 ], "peak_layer": 2, "peak_normalized": 1.9431844817635626, "population_cv": 0.3545327058404385, "rest_mean": 2.266066288332998e-05, "spike_contrast": 0.7678616985591674, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.740025509207044e-05, "top_five_layers": [ 2, 1, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 3.951197868445888e-05, 4.2436669900780544e-05, 3.661722439574078e-05, 3.2650947105139494e-05, 3.204624226782471e-05, 3.01540731015848e-05, 2.8687099984381348e-05, 2.6478855943423696e-05, 2.502457209629938e-05, 2.3117641831049696e-05, 2.197966750827618e-05, 2.1080213628010824e-05, 2.0702012989204377e-05, 1.99326932488475e-05, 1.9394390619709156e-05, 1.8828624888556078e-05, 1.895203058666084e-05, 1.8378550521447323e-05, 1.8309112419956364e-05, 1.8019058188656345e-05, 1.7947008018381894e-05, 1.762924512149766e-05, 1.7346912500215694e-05, 1.7133235814981163e-05, 1.694487400527578e-05, 1.6576635971432552e-05, 1.6019936083466746e-05, 1.5086184248502832e-05, 1.4266219295677729e-05, 1.3403954653767869e-05, 1.2350712495390326e-05, 1.1832595191663131e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.029772771056741476, "first_to_last_ratio": 2.3058080657449813, "imbalance_abs_log_ratio": 0.8354311858351369, "last_quartile_mean": 0.012912076897919178, "mean": 0.01936679895152338, "normalized": [ 1.8092622688469415, 1.9431844502992837, 1.6767106129751548, 1.4950940451422423, 1.4674045233760984, 1.3807616273635772, 1.3135886581915137, 1.2124727235166237, 1.1458806648362485, 1.0585619314023544, 1.0064538755889925, 0.965267610260004, 0.9479497452013365, 0.91272242891937, 0.8880734542201515, 0.8621668660127682, 0.8678176632029413, 0.8415579121933264, 0.838378389023671, 0.8250966015781978, 0.8217974339297908, 0.8072470687715084, 0.7943189250144355, 0.7845346707619233, 0.7759095445302433, 0.7590478000025082, 0.7335564570017248, 0.69079971362327, 0.6532533237931528, 0.6137700206530842, 0.56554185062535, 0.5418171391422113 ], "peak_layer": 2, "peak_normalized": 1.9431844502992837, "population_cv": 0.35453269447871943, "rest_mean": 0.020095702398706367, "spike_contrast": 0.7678617064775293, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.01543072033673525, "top_five_layers": [ 2, 1, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.035039618611335754, 0.0376332625746727, 0.03247251734137535, 0.028955185785889626, 0.028418928384780884, 0.02674093283712864, 0.02544000744819641, 0.023481715470552444, 0.02219204045832157, 0.02050095610320568, 0.019491789862513542, 0.018694143742322922, 0.018358752131462097, 0.01767651177942753, 0.017199140042066574, 0.016697412356734276, 0.01680685020983219, 0.016298282891511917, 0.01623670570552349, 0.015979479998350143, 0.015915585681796074, 0.01563379168510437, 0.015383414924144745, 0.015193925239145756, 0.015026884153485298, 0.014700326137244701, 0.014206640422344208, 0.013378579169511795, 0.012651425786316395, 0.011886760592460632, 0.01095273531973362, 0.010493263602256775 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 1.2918531382410947e-06, "first_to_last_ratio": 1.348602552566167, "imbalance_abs_log_ratio": 0.2990689100691611, "last_quartile_mean": 9.57919837674126e-07, "mean": 1.0475390865849477e-06, "normalized": [ 2.4104888609370185, 1.341971980631633, 1.2193546382838214, 1.0858339683324785, 0.9788660798673142, 0.9472203180914999, 0.9376150883665126, 0.9444624163032238, 0.9288836138476254, 0.8900745462090585, 0.8740769359854389, 0.8711050178927995, 0.8545471342557541, 0.8634403690695975, 0.8786271700589888, 0.8875789012156576, 0.8758221673193468, 0.9032444171738859, 0.9303783636244995, 0.9552629683285443, 0.9757615411235937, 1.0157402642998257, 1.045251506463968, 1.0688091449644375, 1.0624071055149489, 1.0529656438443655, 1.0284038000374691, 0.9523781978486857, 0.8838469105126692, 0.8403571044364565, 0.7713914356307573, 0.7238323895281237 ], "peak_layer": 1, "peak_normalized": 2.4104888609370185, "population_cv": 0.279812062407651, "rest_mean": 1.0410222465132527e-06, "spike_contrast": 1.040064250882766, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.0827300229721004e-06, "top_five_layers": [ 1, 2, 3, 4, 24 ], "top_five_spike_overlap": 1 }, "values": [ 2.5250812996091554e-06, 1.4057681028134539e-06, 1.2773216440109536e-06, 1.1374535233699135e-06, 1.0254004791931948e-06, 9.922503068082733e-07, 9.821884532357217e-07, 9.893612968880916e-07, 9.730418923936668e-07, 9.323878771283489e-07, 9.156297551271564e-07, 9.125165547629877e-07, 8.951715244620573e-07, 9.044875355357362e-07, 9.203963031723106e-07, 9.297735914515215e-07, 9.174579531645577e-07, 9.461838317292859e-07, 9.746077012096066e-07, 1.0006752972913091e-06, 1.0221483535133302e-06, 1.0640276286721928e-06, 1.0949418083328055e-06, 1.1196193554496858e-06, 1.1129129688924877e-06, 1.103022668758058e-06, 1.0772931773317396e-06, 9.976533874578308e-07, 9.258641853193694e-07, 8.803069135865371e-07, 8.08062679880095e-07, 7.582427201668906e-07 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 2.57942979260406e-05, "first_to_last_ratio": 2.5994532681374096, "imbalance_abs_log_ratio": 0.9553011414295731, "last_quartile_mean": 9.922970434672607e-06, "mean": 1.6124295910913133e-05, "normalized": [ 1.8750767487966784, 2.0012252521436076, 1.745806554118046, 1.5722176381119681, 1.5471673963385955, 1.4484453745095827, 1.357829461828116, 1.2499614439371438, 1.1713452937825866, 1.079951327288068, 1.0230197185281116, 0.9769120487640461, 0.96163457802513, 0.918624119720732, 0.8885326600597652, 0.8580725357821459, 0.8699071437806652, 0.8325299891167686, 0.8310410601256768, 0.8095951692808093, 0.8030390759939747, 0.7755468823771603, 0.7515517005264444, 0.7277277649695101, 0.7158336494479539, 0.6950266607011315, 0.6717468100420292, 0.6353600749013578, 0.6043990157307626, 0.5679033620811016, 0.527236654105826, 0.5057328350845046 ], "peak_layer": 2, "peak_normalized": 2.0012252521436076, "population_cv": 0.3945192515369112, "rest_mean": 1.6856638096753698e-05, "spike_contrast": 0.7219498951999067, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.2169648107374087e-05, "top_five_layers": [ 2, 1, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 3.0234292353270575e-05, 3.226834814995527e-05, 2.8149901481810957e-05, 2.5350902433274314e-05, 2.4946984922280535e-05, 2.3355161829385906e-05, 2.189404403907247e-05, 2.0154748199274763e-05, 1.8887118130805902e-05, 1.741345477057621e-05, 1.6495472664246336e-05, 1.575201895320788e-05, 1.550568049424328e-05, 1.4812167137279175e-05, 1.4326963537314441e-05, 1.3835815479978919e-05, 1.4026640201336704e-05, 1.3423959899228066e-05, 1.3399951967585366e-05, 1.305415207752958e-05, 1.2948439689353108e-05, 1.2505147424235474e-05, 1.2118242011638358e-05, 1.1734097824955825e-05, 1.154231358668767e-05, 1.1206815543118864e-05, 1.0831444342329632e-05, 1.0244733857689425e-05, 9.745508577907458e-06, 9.157041858998127e-06, 8.501319825882092e-06, 8.154585884767585e-06 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 1.933050248226209e-05, "first_to_last_ratio": 2.5350541410379415, "imbalance_abs_log_ratio": 0.930214994226497, "last_quartile_mean": 7.6252819098954205e-06, "mean": 1.202020879986776e-05, "normalized": [ 1.8583274986098608, 2.0026775386155595, 1.77731729208076, 1.615557162502337, 1.5930029936690748, 1.4693486681854961, 1.3292468480215616, 1.2198575705059131, 1.1444407348951777, 1.053479376420273, 1.0059382961834429, 0.9516573870963836, 0.9421936613647472, 0.8964225087904136, 0.8652818603489327, 0.841617098226079, 0.8649772378810286, 0.8234169272301419, 0.8325562065776818, 0.8056805753420667, 0.8017546832979361, 0.7661992033221119, 0.7431607861543773, 0.720913207047878, 0.7139696155781592, 0.6926020821557612, 0.6726446950161074, 0.6435870710508805, 0.6274403778164145, 0.5970363543218487, 0.569837411659171, 0.5578570700324221 ], "peak_layer": 2, "peak_normalized": 2.0026775386155595, "population_cv": 0.3945939876253568, "rest_mean": 1.2578481834713818e-05, "spike_contrast": 0.7159476421745723, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.005534411699046e-06, "top_five_layers": [ 2, 1, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 2.2337484551826492e-05, 2.4072602172964253e-05, 2.136372495442629e-05, 1.941933442139998e-05, 1.91482286027167e-05, 1.7661877791397274e-05, 1.5977824659785256e-05, 1.4662942703580484e-05, 1.3756416592514142e-05, 1.2663042070926167e-05, 1.20915883599082e-05, 1.143912049883511e-05, 1.1325364539516158e-05, 1.0775185728562064e-05, 1.0400868632132187e-05, 1.0116413250216283e-05, 1.039720700646285e-05, 9.897643394651823e-06, 1.0007499440689571e-05, 9.68444874160923e-06, 9.63725869951304e-06, 9.209874406224117e-06, 8.932947821449488e-06, 8.66552727529779e-06, 8.582063856010791e-06, 8.325221642735414e-06, 8.08532968221698e-06, 7.736050974926911e-06, 7.5419643508212175e-06, 7.176501640060451e-06, 6.849564670119435e-06, 6.705558462272165e-06 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 7.031241830190993e-05, "first_to_last_ratio": 2.261102148101721, "imbalance_abs_log_ratio": 0.8158523705600311, "last_quartile_mean": 3.109652448074485e-05, "mean": 4.619880877498872e-05, "normalized": [ 1.7997064857450336, 1.904359030499371, 1.6374224610397496, 1.477941957137803, 1.4602990373876177, 1.3729930118356377, 1.3118711870701552, 1.2110312951649602, 1.137824927128826, 1.0604309170276205, 1.0061171703075658, 0.9633053240172016, 0.949330006688662, 0.9216248238136175, 0.906779203920096, 0.8687912686710114, 0.8800028320072268, 0.8475838252231725, 0.8463171144866274, 0.8286039531613457, 0.8201877997739442, 0.8102510580287974, 0.7986912791711968, 0.7937156242787844, 0.7857004463402506, 0.7744091131435279, 0.7453649305566711, 0.6981461341444608, 0.6620680295240863, 0.614979715499695, 0.5681574453782887, 0.5359925918269958 ], "peak_layer": 2, "peak_normalized": 1.904359030499371, "population_cv": 0.3447198298829091, "rest_mean": 4.7895252632938706e-05, "spike_contrast": 0.7733128004546086, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.70380119420588e-05, "top_five_layers": [ 2, 1, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 8.314429578604177e-05, 8.797911868896335e-05, 7.564696716144681e-05, 6.827915785834193e-05, 6.746407598257065e-05, 6.343064160319045e-05, 6.060688610887155e-05, 5.594820322585292e-05, 5.256615622784011e-05, 4.899064515484497e-05, 4.648141475627199e-05, 4.4503558456199244e-05, 4.3857915443368256e-05, 4.2577968997647986e-05, 4.189211904304102e-05, 4.013712168671191e-05, 4.06550825573504e-05, 3.915736306225881e-05, 3.909884253516793e-05, 3.828051558230072e-05, 3.789169932133518e-05, 3.74326336896047e-05, 3.689858567668125e-05, 3.666871634777635e-05, 3.629842467489652e-05, 3.577677853172645e-05, 3.443497189437039e-05, 3.225351974833757e-05, 3.058675429201685e-05, 2.8411330276867375e-05, 2.624819717311766e-05, 2.476221925462596e-05 ] } } }, "mlp_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "c0b8018c349b1bd09e5d17858eb312a2323da0e92de9651e2bb08ae4adf115c7", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.13583384454250336, 0.0415157787501812, 0.018140379339456558, 0.039518751204013824, 0.022877270355820656, 0.015532293356955051, 0.02314678393304348, 0.05432315170764923, 0.03743875026702881, 0.04006272554397583, 0.045583780854940414, 0.05995568633079529, 0.038651175796985626, 0.04559725895524025, 0.05632574483752251, 0.07952121645212173, 0.042337141931056976, 0.05044656619429588, 0.04418695345520973, 0.06310147047042847, 0.029010163620114326, 0.04586305841803551, 0.06033914536237717, 0.07656749337911606, 0.045884523540735245, 0.051604680716991425, 0.06502768397331238, 0.06792232394218445, 0.060306549072265625, 0.08025261014699936, 0.09663844108581543, 0.12230555713176727 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 8.247537380157155e-06, "first_to_last_ratio": 2.3850511827649155, "imbalance_abs_log_ratio": 0.8692205843888265, "last_quartile_mean": 3.458012741930361e-06, "mean": 5.339330805043119e-06, "normalized": [ 1.6764909218312523, 1.9617954794601216, 1.6992151272822646, 1.4983517248278486, 1.5139649706461762, 1.4256520166778042, 1.3385091192829222, 1.2434304822395104, 1.131080096564001, 1.0546107928438107, 1.0037428856625374, 0.962940037127743, 0.9549245789646706, 0.9150321056456306, 0.8876946170122294, 0.8650767880832936, 0.8770248559584294, 0.8519848957954361, 0.8533470944296186, 0.848236507394223, 0.8353817265855179, 0.8173190953401269, 0.8073880080861434, 0.7956133450140085, 0.7777311024585761, 0.7505722139587175, 0.705927759165694, 0.665757931848141, 0.624138138877991, 0.5752620873586926, 0.5511417007015933, 0.530661792875274 ], "peak_layer": 2, "peak_normalized": 1.9617954794601216, "population_cv": 0.3568485733648799, "rest_mean": 5.530472267141535e-06, "spike_contrast": 0.7788063481128092, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.307166909711668e-06, "top_five_layers": [ 2, 3, 1, 5, 4 ], "top_five_spike_overlap": 0 }, "values": [ 8.95133962330874e-06, 1.0474675036675762e-05, 9.07267167349346e-06, 8.000195521162823e-06, 8.08355980552733e-06, 7.612027729919646e-06, 7.146742973418441e-06, 6.639086677751038e-06, 6.039210802555317e-06, 5.630915893561905e-06, 5.359315309760859e-06, 5.141455403645523e-06, 5.098658220958896e-06, 4.885659109277185e-06, 4.7396952140843496e-06, 4.6189311433408875e-06, 4.682725830207346e-06, 4.549029199552024e-06, 4.556302428682102e-06, 4.52901531389216e-06, 4.460379386728164e-06, 4.363937023299513e-06, 4.310911663196748e-06, 4.2480428419366945e-06, 4.1525636333972216e-06, 4.007553343399195e-06, 3.7691818306484492e-06, 3.5547018342185766e-06, 3.3324799915135372e-06, 3.071514584007673e-06, 2.942727860499872e-06, 2.833378857758362e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 3.269888588874892e-05, "first_to_last_ratio": 2.35686858292009, "imbalance_abs_log_ratio": 0.8573338663198418, "last_quartile_mean": 1.3873868965674774e-05, "mean": 2.1262430664137355e-05, "normalized": [ 1.6815888767055769, 1.9686225336479115, 1.7075392779747713, 1.4904980805963362, 1.4972655559630919, 1.4034192556792997, 1.3238794766841318, 1.2301588485530683, 1.121552358208752, 1.0499507133536148, 1.0011277236394078, 0.963324273966619, 0.9560838812000779, 0.9203885412533225, 0.8930508699850467, 0.8703824880945366, 0.8841528709344532, 0.8558102501916335, 0.8550558750774316, 0.8483637687825304, 0.8350347355926667, 0.8178886567183555, 0.8078366810386547, 0.7969743808996977, 0.7797462600958652, 0.7532932557652127, 0.7097594040214169, 0.6713244282550651, 0.6299481338236045, 0.5812721180769863, 0.5573951790036309, 0.5373112462172303 ], "peak_layer": 2, "peak_normalized": 1.9686225336479115, "population_cv": 0.35389080715057486, "rest_mean": 2.202041212984585e-05, "spike_contrast": 0.7797006998811187, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.7169330749311485e-05, "top_five_layers": [ 2, 3, 1, 5, 4 ], "top_five_spike_overlap": 0 }, "values": [ 3.5754666896536946e-05, 4.1857700125547126e-05, 3.630643550422974e-05, 3.169161209370941e-05, 3.183550506946631e-05, 2.9840104616596363e-05, 2.8148895580670796e-05, 2.615616722323466e-05, 2.3846929252613336e-05, 2.232450424344279e-05, 2.1286408809828572e-05, 2.0482615582295693e-05, 2.0328667233115993e-05, 1.9569697542465292e-05, 1.89884322026046e-05, 1.850644730438944e-05, 1.8799239114741795e-05, 1.819660610635765e-05, 1.818056625779718e-05, 1.8038275811704807e-05, 1.7754868167685345e-05, 1.7390300854458474e-05, 1.717657141853124e-05, 1.6945612514973618e-05, 1.6579300790908746e-05, 1.601684562047012e-05, 1.5091210116224829e-05, 1.4273989108914975e-05, 1.339422851742711e-05, 1.2359258107608184e-05, 1.1851576346089132e-05, 1.1424543117755093e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.02899769856594503, "first_to_last_ratio": 2.35686858681256, "imbalance_abs_log_ratio": 0.8573338679713849, "last_quartile_mean": 0.012303485535085201, "mean": 0.01885573595063761, "normalized": [ 1.6815888871597915, 1.9686224147635019, 1.7075392481901774, 1.4904981407970117, 1.4972654379699921, 1.4034192484903574, 1.3238795515861768, 1.230158916579243, 1.1215523220126975, 1.0499507103306343, 1.001127702787151, 0.9633242487753337, 0.9560838749935686, 0.9203885704718483, 0.8930508868493195, 0.8703824268783725, 0.8841529174326813, 0.8558103041085099, 0.8550558906482283, 0.8483637681044575, 0.8350347432767428, 0.8178887044245051, 0.8078366908478906, 0.796974401455123, 0.7797462730931964, 0.7532932002519721, 0.7097594341790385, 0.671324454011456, 0.6299481306480355, 0.5812721137067387, 0.5573951795880433, 0.5373112055882039 ], "peak_layer": 2, "peak_normalized": 1.9686224147635019, "population_cv": 0.35389079505518783, "rest_mean": 0.019527921418624895, "spike_contrast": 0.7797007217053022, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.01522593442350626, "top_five_layers": [ 2, 3, 1, 5, 4 ], "top_five_spike_overlap": 0 }, "values": [ 0.03170759603381157, 0.037119824439287186, 0.03219690918922424, 0.02810443937778473, 0.028232041746377945, 0.026462502777576447, 0.02496272325515747, 0.023195551708340645, 0.021147694438695908, 0.019797593355178833, 0.018876999616622925, 0.01816418766975403, 0.018027665093541145, 0.017354603856801987, 0.016839131712913513, 0.01641170121729374, 0.016671353951096535, 0.016136933118104935, 0.016122708097100258, 0.015996523201465607, 0.015745194628834724, 0.015421893447637558, 0.015232355333864689, 0.015027538873255253, 0.014702689833939075, 0.014203897677361965, 0.013383036479353905, 0.012658316642045975, 0.011878135614097118, 0.010960313491523266, 0.010510096326470375, 0.01013139821588993 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 1.2314776824950968e-06, "first_to_last_ratio": 1.358248310832468, "imbalance_abs_log_ratio": 0.30619586281350625, "last_quartile_mean": 9.066660879852861e-07, "mean": 1.0095481766114744e-06, "normalized": [ 2.2510647132267514, 1.3455279053230764, 1.2283491143156429, 1.0526455105583856, 1.0056690176197902, 0.9692221599384264, 0.9572558258596283, 0.9489099547933215, 0.9218216280115407, 0.8905546788106925, 0.8763660110671972, 0.8694242377636043, 0.8685998082234971, 0.8764693322122661, 0.8891057616302812, 0.8990022944596364, 0.9003146700723137, 0.9275413402074646, 0.9581989480285121, 0.9876176030015524, 1.000156116631407, 1.0336678655116023, 1.068099651182219, 1.0896881962272793, 1.081885225695443, 1.0551976273002857, 0.9988041017347361, 0.9238330401452145, 0.8601167762256441, 0.7958830056879117, 0.7583180274077601, 0.7106898511269164 ], "peak_layer": 1, "peak_normalized": 2.2510647132267514, "population_cv": 0.2561818803841662, "rest_mean": 9.993219375953938e-07, "spike_contrast": 1.0654923375948289, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.0647698672983098e-06, "top_five_layers": [ 1, 2, 3, 24, 25 ], "top_five_spike_overlap": 2 }, "values": [ 2.2725582766724983e-06, 1.3583752433987684e-06, 1.2400776085996768e-06, 1.0626963558024727e-06, 1.015271323012712e-06, 9.784764642972732e-07, 9.663958735472988e-07, 9.579703146300744e-07, 9.306233437200717e-07, 8.990578521661519e-07, 8.847337085171603e-07, 8.777256539360678e-07, 8.768933525971079e-07, 8.848380161907698e-07, 8.975951004686067e-07, 9.075861271412577e-07, 9.089110335480655e-07, 9.363976687382092e-07, 9.673480008132174e-07, 9.970475502996123e-07, 1.0097057838720502e-06, 1.0435375088491128e-06, 1.078298055290361e-06, 1.1000927315762965e-06, 1.092215256903728e-06, 1.0652728406057577e-06, 1.0083408596983645e-06, 9.326539611720364e-07, 8.683293231115385e-07, 8.03482237188291e-07, 7.655585818611144e-07, 7.174756433414586e-07 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 2.5231804784198175e-05, "first_to_last_ratio": 2.6518592370690897, "imbalance_abs_log_ratio": 0.975260992886681, "last_quartile_mean": 9.514760222373297e-06, "mean": 1.575830160049918e-05, "normalized": [ 1.7680072391258728, 2.020274701367778, 1.7747981337484644, 1.5702283057720683, 1.5745659549387958, 1.468575316323847, 1.366589194311488, 1.2663643116143115, 1.1465258163389551, 1.0704343447928568, 1.0164163108220898, 0.9740484512911409, 0.9677541966052103, 0.9245201462352831, 0.8913652028193111, 0.8644158057124367, 0.8839251267896815, 0.845522595189969, 0.8460497665068152, 0.8329662909412521, 0.8145821291394524, 0.7836795820242509, 0.7613757450733065, 0.7366672589275867, 0.716422356677449, 0.689594392508022, 0.652884419046894, 0.6205919149770609, 0.5818976326649927, 0.5425153265661802, 0.5218602418605108, 0.5045817892866661 ], "peak_layer": 2, "peak_normalized": 2.020274701367778, "population_cv": 0.39415073781885135, "rest_mean": 1.645124252212958e-05, "spike_contrast": 0.7304263253994938, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.2016420623695012e-05, "top_five_layers": [ 2, 3, 1, 5, 4 ], "top_five_spike_overlap": 0 }, "values": [ 2.786079130601138e-05, 3.1836098060011864e-05, 2.7967804271611385e-05, 2.47441312239971e-05, 2.4812485207803547e-05, 2.314225275767967e-05, 2.1535124687943608e-05, 1.9955750758526847e-05, 1.8067299606627785e-05, 1.6868227248778567e-05, 1.6016994777601212e-05, 1.5349349268944934e-05, 1.5250162505253684e-05, 1.4568867300113197e-05, 1.4046401702216826e-05, 1.362172497465508e-05, 1.3929158740211278e-05, 1.3324000065040309e-05, 1.3332307389646303e-05, 1.3126134035701398e-05, 1.2836430869356263e-05, 1.2349459211691283e-05, 1.1997988622169942e-05, 1.1608624845393933e-05, 1.128959956986364e-05, 1.0866836419154424e-05, 1.0288349585607648e-05, 9.77947456703987e-06, 9.169718396151438e-06, 8.549120138923172e-06, 8.223631084547378e-06, 7.95135201769881e-06 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 1.902213432458666e-05, "first_to_last_ratio": 2.570514437449803, "imbalance_abs_log_ratio": 0.9441060490896753, "last_quartile_mean": 7.4001274015245144e-06, "mean": 1.1811577877551827e-05, "normalized": [ 1.7836661639182594, 2.007884486098661, 1.801766001704916, 1.6166376372836335, 1.6162792780207194, 1.484195322536222, 1.334217273990145, 1.2390748916935237, 1.1152682343795186, 1.047183670438877, 0.9989511624599826, 0.9468241347561922, 0.942586809823197, 0.9015408911709663, 0.8679632288380377, 0.8434512704540663, 0.8735354304753469, 0.8326703084602718, 0.8456967523674954, 0.8281893546686905, 0.8104136880244638, 0.7723543788975116, 0.750047939286133, 0.727483933767052, 0.7080168021618385, 0.6848377954753971, 0.6576536196741141, 0.6404010158732426, 0.6084114841371344, 0.5799389397954393, 0.5702129585195616, 0.5626451408493897 ], "peak_layer": 2, "peak_normalized": 2.007884486098661, "population_cv": 0.39402258867818274, "rest_mean": 1.2350397237578701e-05, "spike_contrast": 0.720782753960385, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 8.901953333406708e-06, "top_five_layers": [ 2, 3, 1, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 2.1067911802674644e-05, 2.3716283976682462e-05, 2.1281699446262792e-05, 1.909504135255702e-05, 1.9090808564214967e-05, 1.753068863763474e-05, 1.57592112373095e-05, 1.4635429579357151e-05, 1.3173077604733407e-05, 1.2368891475489363e-05, 1.1799189451267011e-05, 1.1183487004018389e-05, 1.1133437510579824e-05, 1.0648620445863344e-05, 1.025201527227182e-05, 9.962490366888233e-06, 1.0317831765860319e-05, 9.835150194703601e-06, 9.989013051381335e-06, 9.782223060028628e-06, 9.572264389134943e-06, 9.12272389541613e-06, 8.859249646775424e-06, 8.592733138357289e-06, 8.36279559734976e-06, 8.089014954748563e-06, 7.767926945234649e-06, 7.564146471850108e-06, 7.186299626482651e-06, 6.849993951618671e-06, 6.735114766343031e-06, 6.645726898568682e-06 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 6.828012237747316e-05, "first_to_last_ratio": 2.312494882445189, "imbalance_abs_log_ratio": 0.8383269774053195, "last_quartile_mean": 2.9526604748753016e-05, "mean": 4.489596511803029e-05, "normalized": [ 1.6577187409379064, 1.9152258678535772, 1.6744752052558183, 1.4840054420576252, 1.4912917773323804, 1.3961062438002583, 1.3297764454700043, 1.2182169964281564, 1.1079961846053805, 1.047881406618081, 0.999963947398217, 0.9691026873070004, 0.9556277478959141, 0.9353258396988691, 0.9030474230504885, 0.8874013307551517, 0.8935396144488046, 0.8678070689766392, 0.8641197392760416, 0.8577613745480667, 0.8343135878548937, 0.8204103169462279, 0.8193971014627366, 0.8081501178832716, 0.7960663239792694, 0.7673062057216204, 0.7200712732146042, 0.6825993169002439, 0.6333797250658595, 0.5852510577399719, 0.5516412156589899, 0.5250226738579302 ], "peak_layer": 2, "peak_normalized": 1.9152258678535772, "population_cv": 0.3428609763636811, "rest_mean": 4.642851807282703e-05, "spike_contrast": 0.7887432268392901, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.6620179162127894e-05, "top_five_layers": [ 2, 3, 1, 5, 4 ], "top_five_spike_overlap": 0 }, "values": [ 7.442488276865333e-05, 8.598591375630349e-05, 7.517718040617183e-05, 6.662585656158626e-05, 6.695298361591995e-05, 6.267953722272068e-05, 5.970159691059962e-05, 5.469302777783014e-05, 4.9744558054953814e-05, 4.704564707935788e-05, 4.489434650167823e-05, 4.350880044512451e-05, 4.2903830035356805e-05, 4.199235627311282e-05, 4.054318560520187e-05, 3.984073919127695e-05, 4.0116323361871764e-05, 3.89610358979553e-05, 3.879548967233859e-05, 3.851002475130372e-05, 3.745731373783201e-05, 3.683311297209002e-05, 3.6787623685086146e-05, 3.628267950261943e-05, 3.574016591301188e-05, 3.4448952646926045e-05, 3.232829476473853e-05, 3.064595512114465e-05, 2.8436194043024443e-05, 2.627541107358411e-05, 2.4766464775893837e-05, 2.357139965170063e-05 ] } } }, "post_attention_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "90a633f12d984935028fe5d7212be48e7da7b4a0806751939a551938d16dc540", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.0933690071105957, 0.35791492462158203, 0.3940892517566681, 0.4057151973247528, 0.09380833059549332, 0.11286329478025436, 0.13231655955314636, 0.1621972620487213, 0.05156813934445381, 0.08533531427383423, 0.12003090232610703, 0.16063299775123596, 0.046871740370988846, 0.079384446144104, 0.11459959298372269, 0.15625107288360596, 0.08397205173969269, 0.11654024571180344, 0.15294702351093292, 0.21572710573673248, 0.03708760067820549, 0.06553087383508682, 0.09390194714069366, 0.12852425873279572, 0.025487789884209633, 0.06059965491294861, 0.08860157430171967, 0.12100546807050705, 0.024166326969861984, 0.06877554953098297, 0.12682415544986725, 0.2025238573551178 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 8.4598292460214e-06, "first_to_last_ratio": 2.328358854359362, "imbalance_abs_log_ratio": 0.8451636650513356, "last_quartile_mean": 3.633387194668103e-06, "mean": 5.48445886039417e-06, "normalized": [ 1.7981292044822659, 1.9357505739332324, 1.6671812164387254, 1.501771337574118, 1.4824621119466825, 1.4013991506965175, 1.3280020709214548, 1.225379388067507, 1.156718780628782, 1.0641415748591472, 1.0106456553801202, 0.9662668896138349, 0.9475625334929744, 0.9090604128642483, 0.8837694050828951, 0.8574965104342549, 0.860785690483099, 0.8373501686260958, 0.8360756725074443, 0.8245861368451315, 0.8217235577025693, 0.8060327700829376, 0.7938754319523651, 0.7839311948498927, 0.7744962255348613, 0.7571785503773947, 0.7308065715532428, 0.6865019323584283, 0.6478069914366877, 0.6078375958541098, 0.5594073652125656, 0.5358673282064139 ], "peak_layer": 2, "peak_normalized": 1.9357505739332324, "population_cv": 0.3564831577662627, "rest_mean": 5.691637507307287e-06, "spike_contrast": 0.7670365797994658, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.365694167063339e-06, "top_five_layers": [ 2, 1, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 9.861765647656284e-06, 1.0616544386721216e-05, 9.143586794380099e-06, 8.236403118644375e-06, 8.130502465064637e-06, 7.68591598898638e-06, 7.283372724486981e-06, 6.720542842231225e-06, 6.343976565403864e-06, 5.836240688950056e-06, 5.542844519368373e-06, 5.2994510042481124e-06, 5.196867732593091e-06, 4.985704435966909e-06, 4.846996944252169e-06, 4.702904334408231e-06, 4.720943707070546e-06, 4.592412551573943e-06, 4.5854226300434675e-06, 4.522408744378481e-06, 4.506709046836477e-06, 4.420653567649424e-06, 4.353977146820398e-06, 4.299438387533883e-06, 4.247692686476512e-06, 4.152714609517716e-06, 4.008078576589469e-06, 3.765091605600901e-06, 3.552870794010232e-06, 3.333660288262763e-06, 3.068046680709813e-06, 2.9389423161774175e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 3.3572764550626744e-05, "first_to_last_ratio": 2.3057895392595262, "imbalance_abs_log_ratio": 0.8354231510987864, "last_quartile_mean": 1.456020333989727e-05, "mean": 2.183873047556517e-05, "normalized": [ 1.80924687671537, 1.9431940819531333, 1.676696643261768, 1.4950864798932781, 1.467401917430866, 1.3807559335924544, 1.31358592866812, 1.212462979303528, 1.1458997558869217, 1.0585578682053984, 1.0064671958441547, 0.9652690219967264, 0.9479597986271983, 0.9127134975291503, 0.888075836142126, 0.862166224116184, 0.867826991667279, 0.841547063844033, 0.83837247628253, 0.8251061585552277, 0.8217963049445071, 0.8072479582581175, 0.7943165572696724, 0.7845285926712973, 0.7759202912841717, 0.7590490167365564, 0.7335626108658329, 0.6907982993808188, 0.6532520589865599, 0.6137777359935852, 0.565539938799635, 0.5418179052937989 ], "peak_layer": 2, "peak_normalized": 1.9431940819531333, "population_cv": 0.3545305270645445, "rest_mean": 2.2660667733968392e-05, "spike_contrast": 0.76786216030628, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.7400269280187784e-05, "top_five_layers": [ 2, 1, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 3.951165490434505e-05, 4.2436891817487776e-05, 3.6616926081478596e-05, 3.265079067205079e-05, 3.204619497410022e-05, 3.0153956686262973e-05, 2.868704905267805e-05, 2.64786522166105e-05, 2.5024995920830406e-05, 2.3117559976526536e-05, 2.197996582253836e-05, 2.1080250007798895e-05, 2.0702238543890417e-05, 1.993250407394953e-05, 1.939444882737007e-05, 1.882861579360906e-05, 1.8952239770442247e-05, 1.8378319509793073e-05, 1.8308990547666326e-05, 1.801927101041656e-05, 1.7946988009498455e-05, 1.762927058734931e-05, 1.7346865206491202e-05, 1.7133108485722914e-05, 1.6945114111877047e-05, 1.6576666894252412e-05, 1.602007614565082e-05, 1.508615787315648e-05, 1.4266195648815483e-05, 1.3404126548266504e-05, 1.2350674296612851e-05, 1.183261520054657e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.029772662557661533, "first_to_last_ratio": 2.3057895177959966, "imbalance_abs_log_ratio": 0.8354231417902483, "last_quartile_mean": 0.01291213370859623, "mean": 0.019366804277524352, "normalized": [ 1.809246767643357, 1.9431941106945074, 1.6766966870594806, 1.495086420690814, 1.4674019077542524, 1.3807559578984756, 1.3135858925156472, 1.2124629647121858, 1.1458997775072552, 1.0585579855568523, 1.0064672559708643, 0.9652689798174655, 0.9479597754699817, 0.9127135219659304, 0.888075806778596, 0.8621662442022726, 0.8678270422679951, 0.841547005089253, 0.8383724840087661, 0.8251060885690371, 0.8217963423351232, 0.8072480008996415, 0.7943165425842438, 0.7845285881999872, 0.7759202953515476, 0.7590489858287814, 0.7335626029644629, 0.6907983214336281, 0.653252038106227, 0.6137777383932114, 0.5655399639078199, 0.5418179038223359 ], "peak_layer": 2, "peak_normalized": 1.9431941106945074, "population_cv": 0.35453052191257556, "rest_mean": 0.020095706399944093, "spike_contrast": 0.7678621746036595, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.015430732816457748, "top_five_layers": [ 2, 1, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.035039328038692474, 0.03763346001505852, 0.03247225657105446, 0.028955046087503433, 0.02841888554394245, 0.02674083039164543, 0.02543996088206768, 0.02348153293132782, 0.022192416712641716, 0.02050088532269001, 0.019492054358124733, 0.01869417540729046, 0.018358951434493065, 0.017676344141364098, 0.017199190333485603, 0.01669740490615368, 0.016807036474347115, 0.016298076137900352, 0.016236595809459686, 0.015979668125510216, 0.01591556891798973, 0.01563381403684616, 0.015383373014628887, 0.015193811617791653, 0.01502709649503231, 0.014700353145599365, 0.014206763356924057, 0.01337855588644743, 0.012651404365897179, 0.01188691332936287, 0.010952701792120934, 0.010493281297385693 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 1.291835857841761e-06, "first_to_last_ratio": 1.3485833026747331, "imbalance_abs_log_ratio": 0.299054636012817, "last_quartile_mean": 9.579206974308363e-07, "mean": 1.0475393885656104e-06, "normalized": [ 2.4104197937214296, 1.3419993768147092, 1.2193873876623726, 1.0857681046985204, 0.9788139215354608, 0.9471793472155275, 0.9376610507921448, 0.9444495548464527, 0.9288857336777386, 0.8900930106166185, 0.8741329012581984, 0.8711139916120108, 0.8545363607420399, 0.8634605233315061, 0.8786425447326217, 0.887541854523117, 0.8758151318715187, 0.9032250016851439, 0.9304164056283383, 0.9552919953757107, 0.9757865467437721, 1.0156631340115454, 1.0452801819870623, 1.0688491005570937, 1.0624669234868362, 1.0529715263673087, 1.0284586355464043, 0.9523438456637114, 0.8838175160851754, 0.8403596296329269, 0.7713843760240694, 0.7237845915529142 ], "peak_layer": 1, "peak_normalized": 2.4104197937214296, "population_cv": 0.27980440532718537, "rest_mean": 1.0410195853991254e-06, "spike_contrast": 1.0400825698677958, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.0827463256646297e-06, "top_five_layers": [ 1, 2, 3, 4, 24 ], "top_five_spike_overlap": 1 }, "values": [ 2.5250096769013908e-06, 1.4057972066439106e-06, 1.2773563184964587e-06, 1.1373848565199296e-06, 1.0253461368847638e-06, 9.922076742441277e-07, 9.82236883828591e-07, 9.893481092149159e-07, 9.730443935040967e-07, 9.324074881078559e-07, 9.156886449090962e-07, 9.12526218144194e-07, 8.951604968387983e-07, 9.045089086612279e-07, 9.204126740769425e-07, 9.297350516135339e-07, 9.174508477372001e-07, 9.46163766002428e-07, 9.746478326633223e-07, 1.000705992737494e-06, 1.0221748425465194e-06, 1.063947138391086e-06, 1.0949721627184772e-06, 1.1196615332664805e-06, 1.1129759514005855e-06, 1.1030291489078081e-06, 1.0773509302453022e-06, 9.976176897907862e-07, 9.258336604034412e-07, 8.80309812600899e-07, 8.080555176093185e-07, 7.581928684885497e-07 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 2.5794245857468923e-05, "first_to_last_ratio": 2.5994382822780953, "imbalance_abs_log_ratio": 0.9552953764086378, "last_quartile_mean": 9.923007610268542e-06, "mean": 1.6124308444886992e-05, "normalized": [ 1.875069425095799, 2.001233172595225, 1.7457889523480976, 1.5722148366283213, 1.547165404000032, 1.4484413155137394, 1.3578243451672811, 1.2499566367465564, 1.1713539721385289, 1.0799457497690719, 1.023025015060715, 0.976913996826235, 0.9616437578274982, 0.9186228415910193, 0.888533041071715, 0.8580748018430988, 0.8699143078938142, 0.8325216144516643, 0.8310392860253453, 0.8096019290349791, 0.8030372108501562, 0.7755478024583314, 0.7515516803716358, 0.7277252251002508, 0.7158404256809536, 0.6950255563813353, 0.6717498413974154, 0.6353591861780038, 0.6043991099627453, 0.567907771476934, 0.5272362442668266, 0.5057355442466751 ], "peak_layer": 2, "peak_normalized": 2.001233172595225, "population_cv": 0.3945173779757133, "rest_mean": 1.685664924648504e-05, "spike_contrast": 0.7219506046728209, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.2169668116257526e-05, "top_five_layers": [ 2, 1, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 3.023419776582159e-05, 3.226850094506517e-05, 2.8149639547336847e-05, 2.5350876967422664e-05, 2.494697218935471e-05, 2.3355114535661414e-05, 2.189397855545394e-05, 2.0154686353635043e-05, 1.8887272744905204e-05, 1.741337837302126e-05, 1.6495570889674127e-05, 1.5752062608953565e-05, 1.550584056531079e-05, 1.4812158042332157e-05, 1.4326980817713775e-05, 1.3835862773703411e-05, 1.402676662110025e-05, 1.3423835298453923e-05, 1.339993377769133e-05, 1.3054271221335512e-05, 1.294841968046967e-05, 1.2505171980592422e-05, 1.2118251106585376e-05, 1.1734065992641263e-05, 1.15424318209989e-05, 1.1206806448171847e-05, 1.0831501640495844e-05, 1.0244727491226513e-05, 9.745517672854476e-06, 9.15712007554248e-06, 8.501319825882092e-06, 8.154635906976182e-06 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 1.9329483848196105e-05, "first_to_last_ratio": 2.5350525368598045, "imbalance_abs_log_ratio": 0.9302143614279234, "last_quartile_mean": 7.624884915458097e-06, "mean": 1.201984051135696e-05, "normalized": [ 1.8583357088751307, 2.0027277021690506, 1.7771668453163478, 1.615570343516219, 1.592938001514604, 1.4694493793664982, 1.32898385240296, 1.2198799651106356, 1.144267264787803, 1.0533998962206053, 1.0058282278662807, 0.9519161926020662, 0.9421831838426558, 0.8966453451440455, 0.8650037408265971, 0.8416374374551236, 0.8651561323934935, 0.8233100437191515, 0.8327440956360556, 0.8058278405503257, 0.8017291581294494, 0.7663098470747268, 0.7431513228704318, 0.7209725235957172, 0.7140172181236788, 0.6925010270670932, 0.6728195124367922, 0.6435648715599169, 0.6274519981756614, 0.5969246531502274, 0.5697514359189264, 0.5578352325817294 ], "peak_layer": 2, "peak_normalized": 2.0027277021690506, "population_cv": 0.3945781723821696, "rest_mean": 1.2578015162876841e-05, "spike_contrast": 0.7159871630405812, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.005697393149603e-06, "top_five_layers": [ 2, 1, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 2.233689883723855e-05, 2.407246756774839e-05, 2.1361262042773888e-05, 1.941889786394313e-05, 1.9146860722685233e-05, 1.766254717949778e-05, 1.597417394805234e-05, 1.4662762623629533e-05, 1.3753910025116056e-05, 1.2661698747251648e-05, 1.20898948807735e-05, 1.144188081525499e-05, 1.1324891602271236e-05, 1.0777534043882042e-05, 1.039720700646285e-05, 1.0116347766597755e-05, 1.039903872879222e-05, 9.896055416902527e-06, 1.0009451216319576e-05, 9.685922123026103e-06, 9.636656614020467e-06, 9.210922144120559e-06, 8.932560376706533e-06, 8.665974746691063e-06, 8.582373084209394e-06, 8.32375189929735e-06, 8.087183232419193e-06, 7.735547114862129e-06, 7.541872946603689e-06, 7.174939128162805e-06, 6.848321390862111e-06, 6.705090527248103e-06 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 7.030938422758481e-05, "first_to_last_ratio": 2.2606879822899777, "imbalance_abs_log_ratio": 0.8156691839012263, "last_quartile_mean": 3.110087936875061e-05, "mean": 4.619929336513451e-05, "normalized": [ 1.8000040071532064, 1.904408036353138, 1.637904058859854, 1.4777606171460849, 1.4597770726472121, 1.372671031064488, 1.3115103960008867, 1.2109361461416157, 1.137534863820107, 1.0602352149716379, 1.0062919047911225, 0.9633358523801769, 0.9491225557491253, 0.9216727196154594, 0.9066025950003278, 0.8688487743640587, 0.8802774784785187, 0.8475087887298653, 0.846208545767534, 0.8286626678374982, 0.8203744064059802, 0.810217596933452, 0.7988918917164441, 0.7937267489942904, 0.7857342550389422, 0.7744580018919649, 0.7454551502724039, 0.6984329250267984, 0.662100300186566, 0.6150609478307189, 0.5682149152709306, 0.5360595335595907 ], "peak_layer": 2, "peak_normalized": 1.904408036353138, "population_cv": 0.3446893198253748, "rest_mean": 4.789507282246767e-05, "spike_contrast": 0.7734007302345921, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.704208429553546e-05, "top_five_layers": [ 2, 1, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 8.315891318488866e-05, 8.798230555839837e-05, 7.567001011921093e-05, 6.82714962749742e-05, 6.744066922692582e-05, 6.341643165796995e-05, 6.0590853536268696e-05, 5.5944394262041897e-05, 5.255330688669346e-05, 4.8982117732521147e-05, 4.6489974920405075e-05, 4.45054356532637e-05, 4.384879139252007e-05, 4.258062836015597e-05, 4.188439925201237e-05, 4.014019941678271e-05, 4.066819747094996e-05, 3.915430716006085e-05, 3.9094236853998154e-05, 3.828362969215959e-05, 3.790071787079796e-05, 3.743148045032285e-05, 3.690824087243527e-05, 3.6669614928541705e-05, 3.630036735557951e-05, 3.5779412428382784e-05, 3.443950117798522e-05, 3.2267107599182054e-05, 3.058856600546278e-05, 2.841538116626907e-05, 2.6251127565046772e-05, 2.4765571652096696e-05 ] } } }, "post_mlp_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "89f2226456bfe96e245d2aad0f699d32c676f9f53099789cd870253d002402f5", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.16915453970432281, 0.374446302652359, 0.39966297149658203, 0.4194087088108063, 0.09892481565475464, 0.11664740741252899, 0.13951337337493896, 0.18593810498714447, 0.06607726216316223, 0.10425819456577301, 0.14551182091236115, 0.19865906238555908, 0.06484340131282806, 0.10430748760700226, 0.14720691740512848, 0.20439592003822327, 0.10052042454481125, 0.14085541665554047, 0.1745898574590683, 0.2420409768819809, 0.049456026405096054, 0.08509506285190582, 0.11783231794834137, 0.15649692714214325, 0.05533855780959129, 0.08507220447063446, 0.11942370235919952, 0.1559232622385025, 0.06628894805908203, 0.12384890764951706, 0.19928614795207977, 0.30097496509552 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 8.247530786320567e-06, "first_to_last_ratio": 2.3850533729557593, "imbalance_abs_log_ratio": 0.86922150268771, "last_quartile_mean": 3.45800680179309e-06, "mean": 5.339328467357518e-06, "normalized": [ 1.676519761737885, 1.9618007671893674, 1.6991821441606432, 1.498358513038152, 1.5139533691039981, 1.4256686527077282, 1.3385028917629673, 1.243419273267211, 1.1310636430692016, 1.054615257538969, 1.0037504793539562, 0.9629245320498924, 0.9549124771530781, 0.9150161537455905, 0.8877036077745802, 0.8650752931079952, 0.8770263471429458, 0.8519858650005171, 0.8533548777823373, 0.8482425851215958, 0.8353975931646603, 0.8173266925801457, 0.8073981560598078, 0.7956149708934007, 0.7777226705206242, 0.7505720315608271, 0.7059216805334544, 0.665764185190204, 0.6241330890536816, 0.5752674068009126, 0.5511430066219813, 0.5306620252116886 ], "peak_layer": 2, "peak_normalized": 1.9618007671893674, "population_cv": 0.3568479486512281, "rest_mean": 5.530464544857817e-06, "spike_contrast": 0.7788122704557785, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.3071936488559e-06, "top_five_layers": [ 2, 3, 1, 5, 4 ], "top_five_spike_overlap": 0 }, "values": [ 8.951489689934533e-06, 1.0474698683538008e-05, 9.072491593542509e-06, 8.000228262972087e-06, 8.083494321908802e-06, 7.612113222421613e-06, 7.14670659363037e-06, 6.639023922616616e-06, 6.039120307832491e-06, 5.630937266687397e-06, 5.359353508538334e-06, 5.141370365890907e-06, 5.098591373098316e-06, 4.885571797785815e-06, 4.739741143566789e-06, 4.618921138899168e-06, 4.6827317419229075e-06, 4.54903238278348e-06, 4.556341991701629e-06, 4.529045781964669e-06, 4.460462150746025e-06, 4.363975676824339e-06, 4.3109639591421e-06, 4.248049663146958e-06, 4.15251679442008e-06, 4.00755061491509e-06, 3.7691477245971328e-06, 3.5547336665331386e-06, 3.332451569804107e-06, 3.0715416414750507e-06, 2.942733544841758e-06, 2.833378857758362e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 3.269887292844942e-05, "first_to_last_ratio": 2.3568639793204684, "imbalance_abs_log_ratio": 0.8573319130483905, "last_quartile_mean": 1.3873890566173941e-05, "mean": 2.12624314031018e-05, "normalized": [ 1.6816001107908207, 1.968609461712434, 1.7075260440141944, 1.4905139409933708, 1.4972595154646264, 1.4034185225085787, 1.3238781474315933, 1.2301608589864579, 1.121553773570522, 1.0499501635664128, 1.001131281922829, 0.9633193641679035, 0.9560899219831485, 0.9203740514079343, 0.8930538331784411, 0.8703809179546808, 0.8841528402061924, 0.8558056863273277, 0.8550540488218812, 0.8483642525948266, 0.8350365031100208, 0.8178976965352276, 0.8078428125234317, 0.7969682791900239, 0.7797478584358322, 0.7532962238158083, 0.7097589943815636, 0.6713244476982704, 0.629948240254275, 0.5812766747709892, 0.5573943041371163, 0.5373112275432649 ], "peak_layer": 2, "peak_normalized": 1.968609461712434, "population_cv": 0.35388972104860866, "rest_mean": 2.2020403102268808e-05, "spike_contrast": 0.7797034481094922, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.716938422759995e-05, "top_five_layers": [ 2, 3, 1, 5, 4 ], "top_five_spike_overlap": 0 }, "values": [ 3.5754907003138214e-05, 4.185742363915779e-05, 3.630615537986159e-05, 3.169195042573847e-05, 3.183537774020806e-05, 2.9840090064681135e-05, 2.8148868295829743e-05, 2.6156210878980346e-05, 2.3846960175433196e-05, 2.2324493329506367e-05, 2.128648520738352e-05, 2.048251189989969e-05, 2.0328796381363645e-05, 1.9569390133256093e-05, 1.8988495867233723e-05, 1.8506414562580176e-05, 1.8799239114741795e-05, 1.819650969991926e-05, 1.8180528059019707e-05, 1.8038286725641228e-05, 1.775490636646282e-05, 1.739049366733525e-05, 1.7176702385768294e-05, 1.6945483366725966e-05, 1.6579335351707414e-05, 1.6016909285099246e-05, 1.5091201930772513e-05, 1.4273990018409677e-05, 1.3394231245911215e-05, 1.2359355423541274e-05, 1.1851558156195097e-05, 1.1424543117755093e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.02899768785573542, "first_to_last_ratio": 2.3568640590123793, "imbalance_abs_log_ratio": 0.8573319468610798, "last_quartile_mean": 0.012303504627197981, "mean": 0.018855736823752522, "normalized": [ 1.6816000706700753, 1.9686094816861437, 1.707526129634271, 1.4905139760038595, 1.4972595403829805, 1.403418590801014, 1.323878107307901, 1.2301608352967786, 1.12155375183925, 1.0499501677926542, 1.0011313114380156, 0.963319363752492, 0.9560898565462423, 0.9203740066049206, 0.8930539078006676, 0.8703809048152793, 0.8841528764919796, 0.8558057204161084, 0.8550539741588212, 0.848364321524986, 0.8350365815065012, 0.8178976558963745, 0.8078427780491525, 0.7969682399430816, 0.7797478175310808, 0.7532962276748001, 0.7097590061776181, 0.6713244229257764, 0.6299482002622888, 0.5812766802470385, 0.5573943141138397, 0.5373111807080079 ], "peak_layer": 2, "peak_normalized": 1.9686094816861437, "population_cv": 0.3538897326937565, "rest_mean": 0.019527913761083728, "spike_contrast": 0.7797034311216169, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.01522598136216402, "top_five_layers": [ 2, 3, 1, 5, 4 ], "top_five_spike_overlap": 0 }, "values": [ 0.03170780837535858, 0.037119582295417786, 0.032196663320064545, 0.028104739263653755, 0.02823193185031414, 0.02646249160170555, 0.02496269717812538, 0.02319558896124363, 0.021147722378373146, 0.019797584041953087, 0.018877068534493446, 0.018164096400141716, 0.01802777871489525, 0.01735433004796505, 0.01683918945491314, 0.0164116732776165, 0.016671353951096535, 0.01613684743642807, 0.016122672706842422, 0.015996534377336502, 0.01574523001909256, 0.015422062948346138, 0.015232470817863941, 0.015027423389256, 0.014702719636261463, 0.014203955419361591, 0.013383029028773308, 0.012658316642045975, 0.011878137476742268, 0.010960400104522705, 0.010510080493986607, 0.01013139821588993 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 1.231474328733384e-06, "first_to_last_ratio": 1.3582476774197694, "imbalance_abs_log_ratio": 0.3061953964681102, "last_quartile_mean": 9.066640416222072e-07, "mean": 1.0095540616816834e-06, "normalized": [ 2.2510689330098104, 1.3453781719433024, 1.2282097485564385, 1.0527316026565288, 1.0056637182516928, 0.969318760716823, 0.9573356047524155, 0.9488541987585868, 0.9217951398143368, 0.8906270200688119, 0.8763270628110001, 0.8694286288822738, 0.8685312885606712, 0.8765222738855435, 0.8891328417344724, 0.8990263889886259, 0.900299568341834, 0.9275418452948997, 0.9582330013794911, 0.9876130845276129, 1.0001282145999768, 1.0337196092881593, 1.0681495050787642, 1.0896942312291085, 1.0818926575150707, 1.0552217685009913, 0.9988175358018173, 0.923772869549939, 0.860068575983845, 0.7959012825140968, 0.7583091587544984, 0.710685708248561 ], "peak_layer": 1, "peak_normalized": 2.2510689330098104, "population_cv": 0.25617417026860584, "rest_mean": 9.993245039704899e-07, "spike_contrast": 1.065513423409031, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.0647936733221285e-06, "top_five_layers": [ 1, 2, 3, 24, 25 ], "top_five_spike_overlap": 2 }, "values": [ 2.2725757844455075e-06, 1.3582319979832391e-06, 1.2399441402521916e-06, 1.0627894653225667e-06, 1.0152718914469006e-06, 9.785796919459244e-07, 9.664820481702918e-07, 9.579196103004506e-07, 9.306020274379989e-07, 8.991361255539232e-07, 8.846995456224249e-07, 8.777352036304364e-07, 8.768292900640517e-07, 8.848966217556153e-07, 8.976276717476139e-07, 9.076157425624842e-07, 9.089010859497648e-07, 9.364036372971896e-07, 9.673880185800954e-07, 9.970488008548273e-07, 1.0096835012518568e-06, 1.043595830196864e-06, 1.0783546713355463e-06, 1.1001052371284459e-06, 1.09222912669793e-06, 1.0653034223651048e-06, 1.008360300147615e-06, 9.325986525254848e-07, 8.682857242092723e-07, 8.035053724597674e-07, 7.655540912310244e-07, 7.174756433414586e-07 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 2.523178955016192e-05, "first_to_last_ratio": 2.651853802016533, "imbalance_abs_log_ratio": 0.9752589433592965, "last_quartile_mean": 9.514773978480662e-06, "mean": 1.5758304414248414e-05, "normalized": [ 1.768014772712438, 2.0202664913584756, 1.7747876589600569, 1.5702354129512541, 1.5745618645822212, 1.4685789787379961, 1.3665832942020837, 1.266364662649249, 1.14652665049396, 1.0704311524660057, 1.0164222471522504, 0.9740486236599476, 0.9677569672847843, 0.9245129976091487, 0.891368391145493, 0.8644143239142155, 0.883928662735923, 0.8455178269951987, 0.8460475954048016, 0.8329649879043416, 0.8145835420025702, 0.783683597592109, 0.7613793029015486, 0.7366657999397114, 0.716423036769208, 0.6895960585495816, 0.6528852259142993, 0.6205906498609965, 0.5818984522075571, 0.54251886575813, 0.5218602063941854, 0.5045816991902581 ], "peak_layer": 2, "peak_normalized": 2.0202664913584756, "population_cv": 0.3941497713845743, "rest_mean": 1.645124066945519e-05, "spike_contrast": 0.7304281104125244, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.2016448636131827e-05, "top_five_layers": [ 2, 3, 1, 5, 4 ], "top_five_spike_overlap": 0 }, "values": [ 2.786091499729082e-05, 3.183597436873242e-05, 2.7967644200543873e-05, 2.4744247639318928e-05, 2.481242518115323e-05, 2.314231460331939e-05, 2.1535035557462834e-05, 1.9955759853473864e-05, 1.8067315977532417e-05, 1.6868179955054075e-05, 1.60170911840396e-05, 1.5349354725913145e-05, 1.5250208889483474e-05, 1.4568757251254283e-05, 1.404645445290953e-05, 1.362170405627694e-05, 1.3929216947872192e-05, 1.3323927305464167e-05, 1.3332275557331741e-05, 1.3126115845807362e-05, 1.2836455425713211e-05, 1.234952469530981e-05, 1.1998046829830855e-05, 1.1608603927015793e-05, 1.1289612302789465e-05, 1.086686461349018e-05, 1.0288364137522876e-05, 9.779456377145834e-06, 9.169732948066667e-06, 8.549177437089384e-06, 8.22363199404208e-06, 7.95135201769881e-06 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 1.9022438777938078e-05, "first_to_last_ratio": 2.5706811468129205, "imbalance_abs_log_ratio": 0.9441709014612467, "last_quartile_mean": 7.3997659342239785e-06, "mean": 1.1811503938474743e-05, "normalized": [ 1.7838575112897632, 2.0081895041574866, 1.8013630165639003, 1.6167453942621992, 1.616253821452643, 1.484346910873369, 1.334088872746376, 1.2391628829890091, 1.1149528907165038, 1.0471548053789776, 0.9987803140821989, 0.9467453609714046, 0.9424805202441363, 0.9017388056265639, 0.868090708413055, 0.8437486142496254, 0.8736503938091038, 0.8327134052809684, 0.8458575878838563, 0.8280086592496597, 0.8103372943527495, 0.7723135523278634, 0.7500419314219764, 0.7274729336065492, 0.7078485216009954, 0.6848456245402912, 0.657709866043606, 0.6402184904077136, 0.6083853394516672, 0.5800023997525418, 0.5702454032899359, 0.5626486629633098 ], "peak_layer": 2, "peak_normalized": 2.0081895041574866, "population_cv": 0.39403850931287454, "rest_mean": 1.2350452262008158e-05, "spike_contrast": 0.7207171691012223, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 8.901182991394307e-06, "top_five_layers": [ 2, 3, 1, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 2.1070040020276792e-05, 2.3719738237559795e-05, 2.1276806364767253e-05, 1.9096194591838866e-05, 1.9090388377662748e-05, 1.7532369383843616e-05, 1.5757595974719152e-05, 1.4636377272836398e-05, 1.3169270459911786e-05, 1.2368473107926548e-05, 1.1797097613452934e-05, 1.1182486559846438e-05, 1.1132112376799341e-05, 1.065089145413367e-05, 1.025345682137413e-05, 9.965940080292057e-06, 1.031912506732624e-05, 9.835597666096874e-06, 9.990850230678916e-06, 9.780027539818548e-06, 9.571302143740468e-06, 9.122184565057978e-06, 8.859123227011878e-06, 8.592549420427531e-06, 8.360755600733683e-06, 8.089056791504845e-06, 7.76854267314775e-06, 7.5619432209350634e-06, 7.185945833043661e-06, 6.850700629001949e-06, 6.735455826856196e-06, 6.645726898568682e-06 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 6.828027426308836e-05, "first_to_last_ratio": 2.3127230006738477, "imbalance_abs_log_ratio": 0.838425618478933, "last_quartile_mean": 2.9523758030336467e-05, "mean": 4.4895063126659807e-05, "normalized": [ 1.6579137880786858, 1.9150708402023993, 1.6745616807855777, 1.4835198878876907, 1.4915155698146907, 1.3960120954471311, 1.3299713054901032, 1.2185230612071285, 1.1078646449578662, 1.0477053876101448, 0.999765654013293, 0.9692189920089408, 0.955488933285625, 0.9355193384387992, 0.9031699366923724, 0.8872109049801036, 0.8933262176117003, 0.8675434009211797, 0.864331741509242, 0.8580824004003949, 0.8343570910048359, 0.8204903297173597, 0.8195590181762232, 0.8083415477202021, 0.7958344380770943, 0.7671025897223802, 0.7201771453938138, 0.6826439856557369, 0.6331928257241357, 0.58535097991478, 0.5516010454074354, 0.5250332221429341 ], "peak_layer": 2, "peak_normalized": 1.9150708402023993, "population_cv": 0.34285892534032564, "rest_mean": 4.642717794922646e-05, "spike_contrast": 0.788797525553888, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.662164308479987e-05, "top_five_layers": [ 2, 3, 1, 5, 4 ], "top_five_spike_overlap": 0 }, "values": [ 7.443214417435229e-05, 8.597722626291215e-05, 7.517955236835405e-05, 6.660271901637316e-05, 6.696168566122651e-05, 6.267405115067959e-05, 5.9709145716624334e-05, 5.470566975418478e-05, 4.9737653171177953e-05, 4.703679951489903e-05, 4.4884542148793116e-05, 4.351314782979898e-05, 4.2896735976682976e-05, 4.200019975542091e-05, 4.0547871321905404e-05, 3.983138958574273e-05, 4.010593693237752e-05, 3.89484157494735e-05, 3.8804228097433224e-05, 3.8523663533851504e-05, 3.745851427083835e-05, 3.683596514747478e-05, 3.6794153857044876e-05, 3.629054481280036e-05, 3.5729037335840985e-05, 3.4439119190210477e-05, 3.233239840483293e-05, 3.064734482904896e-05, 2.8427231882233173e-05, 2.6279369194526225e-05, 2.4764163754298352e-05, 2.357139965170063e-05 ] } } }, "pre_attention_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "ee695e78937de01b7128d3866b8fc5dd328ebdf2c159be37918d4f02952d5248", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.0626930519938469, 0.09030644595623016, 0.20229491591453552, 0.20300470292568207, 0.19049881398677826, 0.12695962190628052, 0.12132706493139267, 0.11585728824138641, 0.13427142798900604, 0.10485932976007462, 0.10419609397649765, 0.10852954536676407, 0.1286265105009079, 0.09096142649650574, 0.10080574452877045, 0.10566592961549759, 0.1287592649459839, 0.09650235623121262, 0.1065765842795372, 0.11446042358875275, 0.12395420670509338, 0.07221997529268265, 0.07677769660949707, 0.08416415005922318, 0.09094838052988052, 0.059439923614263535, 0.062182098627090454, 0.06896103918552399, 0.0776243805885315, 0.06285123527050018, 0.07231374084949493, 0.08016443997621536 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 6.4383448545868305e-06, "first_to_last_ratio": 5.042948145238695, "imbalance_abs_log_ratio": 1.6179908605191557, "last_quartile_mean": 1.276702569441568e-06, "mean": 3.687052599232743e-06, "normalized": [ 3.304965820363612, 2.227169292436081, 1.6521910965438824, 1.3947865539668498, 0.5800793227551304, 1.2714767915022098, 1.750193241091026, 1.7887674026051423, 1.1778703519658833, 1.1179830646270124, 0.8417156564271726, 0.7655624105040819, 0.5268426976300161, 1.047779560839775, 0.8380697743291855, 0.7915788420213868, 0.7099070375965003, 1.3022194692601015, 1.0739447359104763, 1.3540969285822078, 0.6285621976492703, 1.167508624430284, 0.9740363058232884, 0.9425613189042933, 0.35040916273624656, 0.44668469186889276, 0.3295723924580137, 0.4329866871315858, 0.2917160480921282, 0.22501238083060152, 0.3167851028104899, 0.37696503630717276 ], "peak_layer": 1, "peak_normalized": 3.304965820363612, "population_cv": 0.641995540013263, "rest_mean": 3.814996382010981e-06, "spike_contrast": 0.7853627820875961, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.9961561722302575e-06, "top_five_layers": [ 1, 2, 8, 7, 3 ], "top_five_spike_overlap": 0 }, "values": [ 1.218558281834703e-05, 8.211690328607801e-06, 6.091715476941317e-06, 5.142651389178354e-06, 2.1387829747254727e-06, 4.688001808972331e-06, 6.4530545387242455e-06, 6.5952795011980925e-06, 4.342869942774996e-06, 4.122062364331214e-06, 3.1034498988447012e-06, 2.8226688755239593e-06, 1.942496737683541e-06, 3.863218353217235e-06, 3.090007339778822e-06, 2.918592826972599e-06, 2.6174645881837932e-06, 4.80135167890694e-06, 3.959690729971044e-06, 4.992626600142103e-06, 2.3175418846221874e-06, 4.3046657083323225e-06, 3.5913230931328144e-06, 3.475273160802317e-06, 1.2919770142616471e-06, 1.646949954192678e-06, 1.215150746247673e-06, 1.5964446902216878e-06, 1.0755724133559852e-06, 8.296324836010172e-07, 1.1680033367156284e-06, 1.3898899169362267e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 2.5929461344276206e-05, "first_to_last_ratio": 5.201390647867945, "imbalance_abs_log_ratio": 1.6489260221160402, "last_quartile_mean": 4.985101696775018e-06, "mean": 1.4714017034123117e-05, "normalized": [ 3.407292787305506, 2.2558164668132807, 1.6137698282608655, 1.433949322998548, 0.5790123126574623, 1.305711460245382, 1.7258967842855322, 1.7763796775082434, 1.1315333512208903, 1.0856488588758444, 0.8714057435432361, 0.7593394863607054, 0.5216263168993628, 1.0262173986737884, 0.8250528440437026, 0.7676154211606976, 0.723733557926003, 1.3830217582174593, 1.0774149558606307, 1.3018941088272185, 0.646511650324857, 1.1377981887421273, 0.9823485776546648, 0.9506130966721329, 0.33641925298333475, 0.4509311419254927, 0.32185944273605893, 0.4000452689662174, 0.290655292962286, 0.22712751082346105, 0.30970536495859485, 0.3736527695664126 ], "peak_layer": 1, "peak_normalized": 3.407292787305506, "population_cv": 0.6554716761859383, "rest_mean": 1.522972111497775e-05, "spike_contrast": 0.7832851900207317, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.1929214997508097e-05, "top_five_layers": [ 1, 2, 8, 7, 3 ], "top_five_spike_overlap": 0 }, "values": [ 5.0134964112658054e-05, 3.3192121918546036e-05, 2.374503674218431e-05, 2.1099154764669947e-05, 8.519597031408921e-06, 1.921226066770032e-05, 2.539487468311563e-05, 2.6137680833926424e-05, 1.6649401004542597e-05, 1.59742558025755e-05, 1.2821878954127897e-05, 1.1172934136993717e-05, 7.675218512304127e-06, 1.5099780284799635e-05, 1.2139841601310764e-05, 1.1294706382614095e-05, 1.0649027899489738e-05, 2.03498057089746e-05, 1.5853102013352327e-05, 1.9156092093908228e-05, 9.512783435638994e-06, 1.674158193054609e-05, 1.4454293705057353e-05, 1.3987337297294289e-05, 4.9500786190037616e-06, 6.635008503508288e-06, 4.735845323011745e-06, 5.886272901989287e-06, 4.276706931705121e-06, 3.3419580631743884e-06, 4.557010015560081e-06, 5.497933216247475e-06 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.022994505590759218, "first_to_last_ratio": 5.20139065184949, "imbalance_abs_log_ratio": 1.6489260228815172, "last_quartile_mean": 0.004420838027726859, "mean": 0.01304853729379829, "normalized": [ 3.4072926237702874, 2.255816491845768, 1.613769818466143, 1.4339493120942854, 0.5790123351120128, 1.3057114456299215, 1.725896924613148, 1.7763798383286113, 1.13153331687909, 1.0856488688373112, 0.8714057478512427, 0.7593395234826519, 0.5216262984837251, 1.0262174043007157, 0.8250528027479058, 0.7676154484189387, 0.7237335326830917, 1.3830217401378855, 1.0774149116451128, 1.3018940938870247, 0.6465115988335997, 1.1377982160374271, 0.9823485499546105, 0.9506130843152019, 0.3364192535702373, 0.45093116963769325, 0.3218594440675733, 0.4000452442436667, 0.2906553022018551, 0.22712752175948964, 0.30970535698369017, 0.3736527791800823 ], "peak_layer": 1, "peak_normalized": 3.4072926237702874, "population_cv": 0.6554716682951498, "rest_mean": 0.013505868855173941, "spike_contrast": 0.7832851759342461, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.010578946862369775, "top_five_layers": [ 1, 2, 8, 7, 3 ], "top_five_spike_overlap": 0 }, "values": [ 0.04446018487215042, 0.029435105621814728, 0.021057335659861565, 0.018710941076278687, 0.007555264048278332, 0.01703762449324131, 0.022520430386066437, 0.023179158568382263, 0.014764854684472084, 0.014166129752993584, 0.01137057039886713, 0.009908270090818405, 0.0068064602091908455, 0.013390636071562767, 0.010765732266008854, 0.010016258805990219, 0.009443663991987705, 0.018046410754323006, 0.014058688655495644, 0.016987813636660576, 0.008436030708253384, 0.014846602454781532, 0.012818211689591408, 0.01240411028265953, 0.0043897791765630245, 0.005883992183953524, 0.004199794959276915, 0.005220005288720131, 0.0037926265504211187, 0.0029636819381266832, 0.004041201900690794, 0.004875622224062681 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 2.464145257619066e-06, "first_to_last_ratio": 2.649593113901558, "imbalance_abs_log_ratio": 0.9744060862854225, "last_quartile_mean": 9.300089303110326e-07, "mean": 1.2621179932281734e-06, "normalized": [ 9.98073298735891, 2.6081595638248576, 0.448046131778418, 0.5712169168045845, 0.31851900459136234, 0.5125315286968626, 0.5097468218730781, 0.6701588608628778, 0.5743479667542774, 0.6374764102214231, 0.546790403537672, 0.5390341640801587, 0.35740547847929505, 0.6625310250792219, 0.6002964531361203, 0.6405992632336756, 0.4466467073820306, 0.8660901213820976, 0.747225340722879, 0.8852501473817866, 0.3909436983971323, 0.8018997241352653, 0.8266134004412924, 0.9628283052981523, 0.5300152809900267, 0.7412196878662661, 0.7889073080638102, 1.074944484309473, 0.6781360573263348, 0.7166665274120905, 0.6792494446365528, 0.6857707839420146 ], "peak_layer": 1, "peak_normalized": 9.98073298735891, "population_cv": 1.6572947531819389, "rest_mean": 1.3316606755240543e-06, "spike_contrast": 0.6657758429950736, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 8.865875088304165e-07, "top_five_layers": [ 1, 2, 28, 24, 20 ], "top_five_spike_overlap": 1 }, "values": [ 1.259686268895166e-05, 3.2918051147134975e-06, 5.654870847138227e-07, 7.209431487353868e-07, 4.020085668798856e-07, 6.468752644650522e-07, 6.433606358768884e-07, 8.45819556616334e-07, 7.248949032145902e-07, 8.045704475989623e-07, 6.901140068293898e-07, 6.803247174502758e-07, 4.51087885267043e-07, 8.361923278243921e-07, 7.576449547741504e-07, 8.08511856575933e-07, 5.637208460029797e-07, 1.0931079259535181e-06, 9.430865475223982e-07, 1.1172901395184454e-06, 4.934170760861889e-07, 1.012092070595827e-06, 1.0432836461404804e-06, 1.2152029285061872e-06, 6.68941822823399e-07, 9.355067049909849e-07, 9.956941084965365e-07, 1.356706775368366e-06, 8.558877198083792e-07, 9.045177193911513e-07, 8.572929459660372e-07, 8.655236456434068e-07 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 1.6010561125767708e-05, "first_to_last_ratio": 4.917014092262474, "imbalance_abs_log_ratio": 1.5927014544542644, "last_quartile_mean": 3.2561552245624625e-06, "mean": 9.725274459526645e-06, "normalized": [ 2.0495551017102773, 2.324610334145404, 1.6789860752319785, 1.469067342376247, 0.683331640000088, 1.4508358782825064, 1.8574881471610558, 1.6563953320037643, 1.2394147680445438, 1.1757596976660154, 0.9385611021097731, 0.8217156066495895, 0.5863388014820525, 1.1122247987743321, 0.8990502135337656, 0.7981592831866082, 0.7526244364476551, 1.397025330921264, 1.1617275022678442, 1.3251897151639205, 0.6397940246687674, 1.2096957514764108, 1.0918700868694384, 1.0020693481814167, 0.3582271576029711, 0.48201484763579705, 0.332690365953382, 0.356368076645877, 0.30908080698932155, 0.24694487721404904, 0.26695462213676374, 0.32622892746712 ], "peak_layer": 2, "peak_normalized": 2.324610334145404, "population_cv": 0.5441079654169187, "rest_mean": 9.97681458843376e-06, "spike_contrast": 0.83864019815785, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 8.366957763428217e-06, "top_five_layers": [ 2, 1, 7, 3, 8 ], "top_five_spike_overlap": 0 }, "values": [ 1.9932485884055495e-05, 2.2607473511015996e-05, 1.6328600395354442e-05, 1.42870831041364e-05, 6.645587745879311e-06, 1.4109777112025768e-05, 1.8064582036458887e-05, 1.6108899217215367e-05, 1.2053648788423743e-05, 1.143458575825207e-05, 9.127764315053355e-06, 7.991409802343696e-06, 5.7023057706828695e-06, 1.0816691428772174e-05, 8.743510079511907e-06, 7.762318091408815e-06, 7.319479209400015e-06, 1.3586454770120326e-05, 1.1298118806735147e-05, 1.2887833690911066e-05, 6.222172487468924e-06, 1.176462319563143e-05, 1.0618736268952489e-05, 9.745399438543245e-06, 3.4838574265450006e-06, 4.6877266868250445e-06, 3.2355051189369988e-06, 3.465777353994781e-06, 3.0058956781431334e-06, 2.4016067072807346e-06, 2.5962069685192546e-06, 3.1726658562547527e-06 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 9.92899242646672e-06, "first_to_last_ratio": 4.574804953054558, "imbalance_abs_log_ratio": 1.520564064606545, "last_quartile_mean": 2.1703640982195793e-06, "mean": 6.422408389994416e-06, "normalized": [ 1.2694651099378356, 2.2660081023317744, 1.6776221325435599, 1.4243775268645795, 0.778686696628982, 1.544586331363982, 1.9675189416357637, 1.4396717003002808, 1.3121814531183145, 1.2749800755885756, 1.028924014649809, 0.867611835816526, 0.6467747339714943, 1.1792708375805625, 0.9308967888836265, 0.8222106571344964, 0.7916222394567357, 1.4570716548377307, 1.2330890513227566, 1.308271526002021, 0.6046831152920485, 1.2497078701132496, 1.207744340136328, 1.0135338985983424, 0.3691120245165777, 0.5161808316774584, 0.34931346918180795, 0.3250152051676304, 0.32914583570336414, 0.26959120942422543, 0.2525212479205583, 0.29260954229900277 ], "peak_layer": 2, "peak_normalized": 2.2660081023317744, "population_cv": 0.5120965787375209, "rest_mean": 6.5544765961762836e-06, "spike_contrast": 0.8710443912398678, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.7092400766123316e-06, "top_five_layers": [ 2, 7, 3, 6, 18 ], "top_five_spike_overlap": 0 }, "values": [ 8.153023372869939e-06, 1.4553229448210914e-05, 1.0774374459288083e-05, 9.147934179054573e-06, 5.001043973607011e-06, 9.919964213622734e-06, 1.2636210158234462e-05, 9.24615960684605e-06, 8.427365173702128e-06, 8.188442734535784e-06, 6.608170224353671e-06, 5.572157533606514e-06, 4.153851477894932e-06, 7.5737589213531464e-06, 5.978599347145064e-06, 5.280572622723412e-06, 5.084121312393108e-06, 9.357909220852889e-06, 7.919401468825527e-06, 8.402254024986178e-06, 3.883521912939614e-06, 8.026134310057387e-06, 7.756627383059822e-06, 6.509328613901744e-06, 2.3705881631030934e-06, 3.3151241041196045e-06, 2.2434337552112993e-06, 2.087380380544346e-06, 2.1139089767530095e-06, 1.731424845274887e-06, 1.6217945812968537e-06, 1.8792579794535413e-06 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 4.949699314238387e-05, "first_to_last_ratio": 5.262607641337885, "imbalance_abs_log_ratio": 1.6606266532103107, "last_quartile_mean": 9.405412015439651e-06, "mean": 2.8804784136582384e-05, "normalized": [ 2.499900864670067, 2.416668640808701, 1.6691856383541526, 1.5215442251975408, 0.6043399354947968, 1.3939313584258168, 1.817946933469592, 1.823363590450555, 1.1883363380697656, 1.0810267152424702, 0.8648382834598256, 0.8031803638827597, 0.533176912017834, 1.0261239559732709, 0.8554824003261805, 0.7869370867194827, 0.7081002132573654, 1.3561635382832782, 1.1101608226630673, 1.398379059233539, 0.6739882703135641, 1.211767597856563, 0.9998718364198766, 1.043404895274106, 0.34963454859286996, 0.4704717001014101, 0.31939467298460794, 0.36230101093546613, 0.28800141494502707, 0.22353921217823322, 0.2657184225315432, 0.3331195418666723 ], "peak_layer": 1, "peak_normalized": 2.499900864670067, "population_cv": 0.5921886777524846, "rest_mean": 2.957433365803876e-05, "spike_contrast": 0.8334665120685795, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.4649216720717958e-05, "top_five_layers": [ 1, 2, 8, 7, 3 ], "top_five_spike_overlap": 0 }, "values": [ 7.200910476967692e-05, 6.961161852814257e-05, 4.8080531996674836e-05, 4.382775296107866e-05, 1.7407881387043744e-05, 4.01518918806687e-05, 5.2365568990353495e-05, 5.2521594625432044e-05, 3.4229771699756384e-05, 3.113874117843807e-05, 2.4911480068112724e-05, 2.3135437004384585e-05, 1.5358045857283287e-05, 2.9557279049186036e-05, 2.4641985874040984e-05, 2.2667552912025712e-05, 2.0396673789946362e-05, 3.906399797415361e-05, 3.197794285370037e-05, 4.028000694233924e-05, 1.941408663697075e-05, 3.490470407996327e-05, 2.880109241232276e-05, 3.0055052775423974e-05, 1.0071147698909044e-05, 1.3551835763792042e-05, 9.200094609695952e-06, 1.0436002412461676e-05, 8.295818588521797e-06, 6.438998752855696e-06, 7.65396180213429e-06, 9.595436495146714e-06 ] } } }, "pre_mlp_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "e53fa425d89dfeb0e9395eca9bcd73dc0a55314abf45c5b10b43e2cbfa724345", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.059514161199331284, 0.08157797157764435, 0.14292451739311218, 0.10997853428125381, 0.0991024449467659, 0.1065012738108635, 0.12155740708112717, 0.1095810979604721, 0.06964563578367233, 0.08311934769153595, 0.09363401681184769, 0.0941167026758194, 0.08564268052577972, 0.09493856877088547, 0.09063401818275452, 0.09473779052495956, 0.10505300015211105, 0.09355910867452621, 0.09618847072124481, 0.11416846513748169, 0.047205954790115356, 0.053352247923612595, 0.06512638926506042, 0.07829347252845764, 0.05368548259139061, 0.05310077592730522, 0.061235878616571426, 0.07320034503936768, 0.05304349213838577, 0.0639304369688034, 0.07404901832342148, 0.08157160878181458 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 2.6960831434053034e-06, "first_to_last_ratio": 0.38455700861487224, "imbalance_abs_log_ratio": 0.9556632341481542, "last_quartile_mean": 7.0108802674440085e-06, "mean": 4.081287496404684e-06, "normalized": [ 1.6016557623725205, 0.5624254649150265, 0.3792516557450063, 0.6472346627621453, 0.3825774509297101, 0.309883335456179, 0.41924156145633357, 0.9824999654437843, 0.9295888075476517, 0.6942195909151134, 0.6392271159962603, 0.6622046687372564, 0.5739498407816973, 0.6102779288338825, 0.725372269896936, 0.7725414351920691, 0.5151773567621832, 0.6691928665493493, 0.6340666932552725, 0.5067672404198594, 0.8579891398085714, 1.216818523393502, 1.4001261788695554, 1.56522304857102, 1.4389835581991173, 1.8374458775865694, 2.199543673756739, 1.6451073151800026, 2.0248581159734593, 1.9770863776726855, 1.3258318688286215, 1.2936306481919198 ], "peak_layer": 27, "peak_normalized": 2.199543673756739, "population_cv": 0.5384151707957241, "rest_mean": 3.857702406801562e-06, "spike_contrast": 1.3709318196595643, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.288646980261546e-06, "top_five_layers": [ 27, 29, 30, 26, 28 ], "top_five_spike_overlap": 0 }, "values": [ 6.5368176365154795e-06, 2.295420017617289e-06, 1.5478350405828678e-06, 2.6415507363708457e-06, 1.5614085668858024e-06, 1.2647229823414818e-06, 1.7110453427449102e-06, 4.009864824183751e-06, 3.793919177041971e-06, 2.833309736161027e-06, 2.6088696358783636e-06, 2.7026476345781703e-06, 2.3424543087458005e-06, 2.4907196802814724e-06, 2.960452775369049e-06, 3.1529636999039212e-06, 2.1025869045843137e-06, 2.7311684789310675e-06, 2.587808467069408e-06, 2.068262801913079e-06, 3.501700348351733e-06, 4.966186224919511e-06, 5.7143174672091845e-06, 6.388125257217325e-06, 5.872905603609979e-06, 7.499144885514397e-06, 8.976970093499403e-06, 6.714155915688025e-06, 8.264028110716026e-06, 8.06905791250756e-06, 5.411101028585108e-06, 5.279678589431569e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 1.0839275830676343e-05, "first_to_last_ratio": 0.38713025500084114, "imbalance_abs_log_ratio": 0.9489940663518645, "last_quartile_mean": 2.7999040867143776e-05, "mean": 1.6318590368769037e-05, "normalized": [ 1.6422297380818347, 0.577857691983563, 0.3782290994995378, 0.6502872006236952, 0.3822780102727073, 0.31178153988993046, 0.41970720095668634, 0.9514590030920431, 0.920750817583778, 0.6902360262942715, 0.6326124977684876, 0.6666642010329482, 0.573439016269761, 0.6073224038931471, 0.7243939579070396, 0.7687525976401163, 0.5168423803555005, 0.6726938023393403, 0.6400255756173949, 0.5037402338382693, 0.8679547154047897, 1.2120933210153013, 1.403510673963058, 1.5589326681275335, 1.4338206424993105, 1.8383886875461704, 2.207829035366123, 1.646625228594747, 2.022240419628589, 1.965711109970758, 1.3282014630518373, 1.283389039891731 ], "peak_layer": 27, "peak_normalized": 2.207829035366123, "population_cv": 0.5389668648883295, "rest_mean": 1.542631884553379e-05, "spike_contrast": 1.3701814934519443, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.1136856594239363e-05, "top_five_layers": [ 27, 29, 30, 26, 28 ], "top_five_spike_overlap": 0 }, "values": [ 2.6798874387168325e-05, 9.429822966922075e-06, 6.1721657402813435e-06, 1.061177044903161e-05, 6.238238256628392e-06, 5.087835234007798e-06, 6.849029887234792e-06, 1.5526469724136405e-05, 1.5025355423858855e-05, 1.1263678970863111e-05, 1.0323344213247765e-05, 1.0879020010179374e-05, 9.357716407976113e-06, 9.910645530908369e-06, 1.18210882646963e-05, 1.254495873581618e-05, 8.434139090240933e-06, 1.097741460398538e-05, 1.044431519403588e-05, 8.220330528274644e-06, 1.4163797459332272e-05, 1.9779654394369572e-05, 2.2903315766598098e-05, 2.5439583623665385e-05, 2.339793172723148e-05, 2.9999911930644885e-05, 3.602865763241425e-05, 2.687060259631835e-05, 3.300011303508654e-05, 3.2077634386951104e-05, 2.1674375602742657e-05, 2.094310002576094e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.009612377849407494, "first_to_last_ratio": 0.38713024225946485, "imbalance_abs_log_ratio": 0.9489940992642404, "last_quartile_mean": 0.024829829344525933, "mean": 0.014471489033894613, "normalized": [ 1.6422296302631012, 0.5778576663317349, 0.37822910199946935, 0.6502872213280481, 0.38227800767762105, 0.3117815365380894, 0.4197072091098399, 0.9514589510980404, 0.9207507998752875, 0.6902360121427672, 0.6326125119311165, 0.6666641983580384, 0.573439005071423, 0.607322401384804, 0.7243939498038978, 0.7687525851502272, 0.5168424285086333, 0.6726938110990978, 0.6400255788101445, 0.5037402548123486, 0.8679547482284273, 1.2120933573235502, 1.4035107155843847, 1.5589326526958214, 1.4338205694791435, 1.8383887153176988, 2.2078290833379297, 1.6466252521885574, 2.022240548084217, 1.9657110301441079, 1.3282014367981623, 1.28338902952427 ], "peak_layer": 27, "peak_normalized": 2.2078290833379297, "population_cv": 0.5389668665215603, "rest_mean": 0.01368021371739882, "spike_contrast": 1.3701814993673933, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.018744375742971896, "top_five_layers": [ 27, 29, 30, 26, 28 ], "top_five_spike_overlap": 0 }, "values": [ 0.023765508085489273, 0.008362460881471634, 0.005473538301885128, 0.009410624392330647, 0.0055321319960057735, 0.004511943086981773, 0.006073788274079561, 0.013769027777016163, 0.013324635103344917, 0.009988742880523205, 0.009154845029115677, 0.009647623635828495, 0.008298516273498535, 0.008788859471678734, 0.010483059100806713, 0.011124994605779648, 0.007479479536414146, 0.009734881110489368, 0.00926212314516306, 0.007289871573448181, 0.012560597620904446, 0.017540795728564262, 0.020310889929533005, 0.022560076788067818, 0.020749518647789955, 0.026604222133755684, 0.03195057436823845, 0.02382911927998066, 0.02926483191549778, 0.02844676561653614, 0.019221052527427673, 0.018572550266981125 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 6.596527093449822e-07, "first_to_last_ratio": 0.3487620963761673, "imbalance_abs_log_ratio": 1.053365261498487, "last_quartile_mean": 1.8914116992618801e-06, "mean": 1.0618384331806396e-06, "normalized": [ 1.9623284636711533, 0.5051262881892306, 0.3546331093387114, 0.6524291934355505, 0.33242707700757573, 0.26547608492472724, 0.31949906663212685, 0.5779720956168373, 0.775638331440552, 0.5998773794374831, 0.5826887294747466, 0.6213115726756598, 0.5572960370183113, 0.6050525236179888, 0.7278412668111575, 0.7634285200712856, 0.5211325543229903, 0.6528362049921996, 0.6569013419877625, 0.48939266573007467, 0.8730686968435768, 1.2548058690120942, 1.4927712810480218, 1.6059753116097122, 1.4417731559107498, 1.8108704859544356, 2.3560676855554643, 1.7166277442710838, 2.056050650260877, 2.0539911278871554, 1.42644194116846, 1.3882675440822447 ], "peak_layer": 27, "peak_normalized": 2.3560676855554643, "population_cv": 0.600205151665452, "rest_mean": 9.962249070785792e-07, "spike_contrast": 1.4215178360523206, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.416151474131766e-06, "top_five_layers": [ 27, 29, 30, 1, 26 ], "top_five_spike_overlap": 0 }, "values": [ 2.0836757812503492e-06, 5.363625064092048e-07, 3.765630651741958e-07, 6.927743925189134e-07, 3.52983846596544e-07, 2.818927100634028e-07, 3.3925638831533433e-07, 6.137129844319134e-07, 8.236025905716815e-07, 6.369728566824051e-07, 6.187212875374826e-07, 6.597325068469218e-07, 5.917583507653035e-07, 6.424680236705171e-07, 7.728498303549713e-07, 8.106377435979084e-07, 5.533585749617487e-07, 6.932065730325121e-07, 6.975230917305453e-07, 5.19655941388919e-07, 9.270578971154464e-07, 1.332401097897673e-06, 1.5850819181650877e-06, 1.7052863086064463e-06, 1.5309301488741767e-06, 1.9228518794989213e-06, 2.50176321969775e-06, 1.8227813143312233e-06, 2.183193601013045e-06, 2.181006721002632e-06, 1.5146508758334676e-06, 1.4741158338438254e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 7.227550952393358e-06, "first_to_last_ratio": 0.3980164784812622, "imbalance_abs_log_ratio": 0.9212618713355372, "last_quartile_mean": 1.8158923921873793e-05, "mean": 1.0911241304256691e-05, "normalized": [ 1.6660127430322156, 0.5876271935840073, 0.29483709953854387, 0.5894374739528777, 0.41635452404712836, 0.33657065533584085, 0.43430341472813805, 0.9740163595711268, 0.9875415328473797, 0.7356703350916547, 0.6756170470367248, 0.7197599061272355, 0.6198568191216928, 0.6512295663928718, 0.7665848858812202, 0.8116271754128533, 0.5305907755231244, 0.6901913960659463, 0.6428575406488269, 0.5176364098703187, 0.8723657576961961, 1.2199632610317463, 1.415147460346549, 1.5302808917079092, 1.4588334149411595, 1.8140206258329823, 2.1102391081606, 1.5869337150428715, 1.998597539089171, 1.88124956086161, 1.2686189428642605, 1.195426868615217 ], "peak_layer": 27, "peak_normalized": 2.1102391081606, "population_cv": 0.5159943803714161, "rest_mean": 1.0306438963636066e-05, "spike_contrast": 1.375564731293613, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.4177173943608068e-05, "top_five_layers": [ 27, 29, 30, 26, 1 ], "top_five_spike_overlap": 0 }, "values": [ 1.81782670551911e-05, 6.411742106138263e-06, 3.2170387385122012e-06, 6.431494512071367e-06, 4.542944679997163e-06, 3.672403636301169e-06, 4.738789357361384e-06, 1.0627727533574216e-05, 1.0775303962873295e-05, 8.027076546568424e-06, 7.371820629487047e-06, 7.85347401688341e-06, 6.763407327525783e-06, 7.105722943379078e-06, 8.364392670046072e-06, 8.855859960021917e-06, 5.789403985545505e-06, 7.530844868597342e-06, 7.014373750280356e-06, 5.648055775964167e-06, 9.51859328779392e-06, 1.3311313523445278e-05, 1.5441015420947224e-05, 1.66972640727181e-05, 1.591768341313582e-05, 1.979321677936241e-05, 2.3025328118819743e-05, 1.7315416698693298e-05, 2.180718001909554e-05, 2.0526767912087962e-05, 1.384220740874298e-05, 1.3043591025052592e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 4.632263923554092e-06, "first_to_last_ratio": 0.3802213248794799, "imbalance_abs_log_ratio": 0.9670017619171267, "last_quartile_mean": 1.218307238559646e-05, "mean": 7.265074810192118e-06, "normalized": [ 1.6008181275970341, 0.5739917816715392, 0.235460636588888, 0.49956890329522624, 0.4370066400923714, 0.3438728691875936, 0.4329980193133239, 0.9771405297973998, 0.9929176905310494, 0.7133552305353121, 0.6552622145802756, 0.7166704391631107, 0.6237781231575423, 0.6649176564292614, 0.7618691062673217, 0.8379753775300484, 0.5325892299976221, 0.6974715354559842, 0.6405465790924313, 0.531438133277865, 0.87605151631772, 1.2349821890311432, 1.4485483745005174, 1.55527350554067, 1.5475592171159964, 1.891436311373116, 2.0687037039687666, 1.6168445987295306, 1.977149264052817, 1.9131038449622708, 1.2696779592298815, 1.1310206916163696 ], "peak_layer": 27, "peak_normalized": 2.0687037039687666, "population_cv": 0.5243879132228086, "rest_mean": 6.817757480348519e-06, "spike_contrast": 1.4199080004313516, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.680588391347555e-06, "top_five_layers": [ 27, 29, 30, 26, 28 ], "top_five_spike_overlap": 0 }, "values": [ 1.1630063454504125e-05, 4.170093234279193e-06, 1.7106391396737308e-06, 3.6294054552854504e-06, 3.1748859328217804e-06, 2.4982621198432753e-06, 3.1457630029763095e-06, 7.098999049048871e-06, 7.21362130207126e-06, 5.182579116080888e-06, 4.760529009217862e-06, 5.206664354773238e-06, 4.531794729700778e-06, 4.8306765165762044e-06, 5.535036052606301e-06, 6.0879538068547845e-06, 3.869300599035341e-06, 5.067182883067289e-06, 4.653618816519156e-06, 3.860937795252539e-06, 6.364579803630477e-06, 8.972237992566079e-06, 1.0523812306928448e-05, 1.1299178368062712e-05, 1.1243133485550061e-05, 1.3741426300839521e-05, 1.502928716945462e-05, 1.1746496966225095e-05, 1.4364137314260006e-05, 1.3898842553317081e-05, 9.224305358657148e-06, 8.216949936468154e-06 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 2.248317332487204e-05, "first_to_last_ratio": 0.36732053671673337, "imbalance_abs_log_ratio": 1.001520415068512, "last_quartile_mean": 6.120859325164929e-05, "mean": 3.520552513691655e-05, "normalized": [ 1.6113007958889953, 0.5636281288577828, 0.3099745506097867, 0.6128259302643165, 0.3776724202532394, 0.30228081296349946, 0.41660627301599373, 0.9147212761628488, 0.9143135144743936, 0.6818048236562128, 0.6378236024718279, 0.669448797145322, 0.5777616284561341, 0.6047361595032282, 0.735292441295308, 0.776547208280745, 0.5187662829625476, 0.6862775953042356, 0.6442540642333291, 0.503974019865995, 0.8786895478652295, 1.1802818108868638, 1.3947424485633424, 1.5774151382314772, 1.403240136016062, 1.8387857304360302, 2.291845133717591, 1.6909973322209617, 2.0609348356508104, 1.9921347125886224, 1.3276799111046451, 1.3032429370526224 ], "peak_layer": 27, "peak_normalized": 2.291845133717591, "population_cv": 0.5527267740898714, "rest_mean": 3.3335239331117247e-05, "spike_contrast": 1.3590743428664134, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.530506848823279e-05, "top_five_layers": [ 27, 29, 30, 26, 28 ], "top_five_spike_overlap": 0 }, "values": [ 5.672669067280367e-05, 1.9842824258375913e-05, 1.0912816833297256e-05, 2.1574858692474663e-05, 1.329615588474553e-05, 1.064195475919405e-05, 1.4666842616861686e-05, 3.220324288122356e-05, 3.2188887416850775e-05, 2.4003296857699752e-05, 2.2454914869740605e-05, 2.356829645577818e-05, 2.0340401533758268e-05, 2.1290054064593278e-05, 2.5886356525006704e-05, 2.733875226113014e-05, 1.826343941502273e-05, 2.416076313238591e-05, 2.2681302652927116e-05, 1.7742670024745166e-05, 3.093472696491517e-05, 4.155244096182287e-05, 4.910264033242129e-05, 5.553372830036096e-05, 4.940180588164367e-05, 6.473541725426912e-05, 8.068561146501452e-05, 5.953244908596389e-05, 7.255629316205159e-05, 7.013414870016277e-05, 4.6741668484173715e-05, 4.588135197991505e-05 ] } } } }, "selector": { "expected_uniform_count": 65, "identities_unique": true, "identity_order_sha256": "97fd9721a263024f6a746638f32e0ad9a988cd4c111c00e9e178c1022e5c9d5e", "passed": true, "uniform_count": 65, "uniform_indices": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64 ], "visit_count": 65, "visits": [ { "branch": "attention", "coefficient": "uniform", "group": 1, "index": 0, "kind": "depth", "layer": 1, "offset": 1, "sources": 1, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 1, "index": 1, "kind": "depth", "layer": 1, "offset": 1, "sources": 2, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 1, "index": 2, "kind": "depth", "layer": 2, "offset": 2, "sources": 2, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 1, "index": 3, "kind": "depth", "layer": 2, "offset": 2, "sources": 2, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 1, "index": 4, "kind": "depth", "layer": 3, "offset": 3, "sources": 2, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 1, "index": 5, "kind": "depth", "layer": 3, "offset": 3, "sources": 2, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 1, "index": 6, "kind": "depth", "layer": 4, "offset": 4, "sources": 2, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 1, "index": 7, "kind": "depth", "layer": 4, "offset": 4, "sources": 2, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 2, "index": 8, "kind": "depth", "layer": 5, "offset": 1, "sources": 2, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 2, "index": 9, "kind": "depth", "layer": 5, "offset": 1, "sources": 3, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 2, "index": 10, "kind": "depth", "layer": 6, "offset": 2, "sources": 3, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 2, "index": 11, "kind": "depth", "layer": 6, "offset": 2, "sources": 3, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 2, "index": 12, "kind": "depth", "layer": 7, "offset": 3, "sources": 3, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 2, "index": 13, "kind": "depth", "layer": 7, "offset": 3, "sources": 3, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 2, "index": 14, "kind": "depth", "layer": 8, "offset": 4, "sources": 3, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 2, "index": 15, "kind": "depth", "layer": 8, "offset": 4, "sources": 3, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 3, "index": 16, "kind": "depth", "layer": 9, "offset": 1, "sources": 3, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 3, "index": 17, "kind": "depth", "layer": 9, "offset": 1, "sources": 4, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 3, "index": 18, "kind": "depth", "layer": 10, "offset": 2, "sources": 4, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 3, "index": 19, "kind": "depth", "layer": 10, "offset": 2, "sources": 4, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 3, "index": 20, "kind": "depth", "layer": 11, "offset": 3, "sources": 4, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 3, "index": 21, "kind": "depth", "layer": 11, "offset": 3, "sources": 4, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 3, "index": 22, "kind": "depth", "layer": 12, "offset": 4, "sources": 4, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 3, "index": 23, "kind": "depth", "layer": 12, "offset": 4, "sources": 4, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 4, "index": 24, "kind": "depth", "layer": 13, "offset": 1, "sources": 4, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 4, "index": 25, "kind": "depth", "layer": 13, "offset": 1, "sources": 5, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 4, "index": 26, "kind": "depth", "layer": 14, "offset": 2, "sources": 5, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 4, "index": 27, "kind": "depth", "layer": 14, "offset": 2, "sources": 5, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 4, "index": 28, "kind": "depth", "layer": 15, "offset": 3, "sources": 5, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 4, "index": 29, "kind": "depth", "layer": 15, "offset": 3, "sources": 5, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 4, "index": 30, "kind": "depth", "layer": 16, "offset": 4, "sources": 5, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 4, "index": 31, "kind": "depth", "layer": 16, "offset": 4, "sources": 5, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 5, "index": 32, "kind": "depth", "layer": 17, "offset": 1, "sources": 5, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 5, "index": 33, "kind": "depth", "layer": 17, "offset": 1, "sources": 6, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 5, "index": 34, "kind": "depth", "layer": 18, "offset": 2, "sources": 6, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 5, "index": 35, "kind": "depth", "layer": 18, "offset": 2, "sources": 6, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 5, "index": 36, "kind": "depth", "layer": 19, "offset": 3, "sources": 6, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 5, "index": 37, "kind": "depth", "layer": 19, "offset": 3, "sources": 6, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 5, "index": 38, "kind": "depth", "layer": 20, "offset": 4, "sources": 6, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 5, "index": 39, "kind": "depth", "layer": 20, "offset": 4, "sources": 6, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 6, "index": 40, "kind": "depth", "layer": 21, "offset": 1, "sources": 6, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 6, "index": 41, "kind": "depth", "layer": 21, "offset": 1, "sources": 7, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 6, "index": 42, "kind": "depth", "layer": 22, "offset": 2, "sources": 7, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 6, "index": 43, "kind": "depth", "layer": 22, "offset": 2, "sources": 7, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 6, "index": 44, "kind": "depth", "layer": 23, "offset": 3, "sources": 7, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 6, "index": 45, "kind": "depth", "layer": 23, "offset": 3, "sources": 7, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 6, "index": 46, "kind": "depth", "layer": 24, "offset": 4, "sources": 7, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 6, "index": 47, "kind": "depth", "layer": 24, "offset": 4, "sources": 7, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 7, "index": 48, "kind": "depth", "layer": 25, "offset": 1, "sources": 7, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 7, "index": 49, "kind": "depth", "layer": 25, "offset": 1, "sources": 8, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 7, "index": 50, "kind": "depth", "layer": 26, "offset": 2, "sources": 8, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 7, "index": 51, "kind": "depth", "layer": 26, "offset": 2, "sources": 8, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 7, "index": 52, "kind": "depth", "layer": 27, "offset": 3, "sources": 8, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 7, "index": 53, "kind": "depth", "layer": 27, "offset": 3, "sources": 8, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 7, "index": 54, "kind": "depth", "layer": 28, "offset": 4, "sources": 8, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 7, "index": 55, "kind": "depth", "layer": 28, "offset": 4, "sources": 8, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 8, "index": 56, "kind": "depth", "layer": 29, "offset": 1, "sources": 8, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 8, "index": 57, "kind": "depth", "layer": 29, "offset": 1, "sources": 9, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 8, "index": 58, "kind": "depth", "layer": 30, "offset": 2, "sources": 9, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 8, "index": 59, "kind": "depth", "layer": 30, "offset": 2, "sources": 9, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 8, "index": 60, "kind": "depth", "layer": 31, "offset": 3, "sources": 9, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 8, "index": 61, "kind": "depth", "layer": 31, "offset": 3, "sources": 9, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 8, "index": 62, "kind": "depth", "layer": 32, "offset": 4, "sources": 9, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 8, "index": 63, "kind": "depth", "layer": 32, "offset": 4, "sources": 9, "use_uniform": true }, { "branch": "output", "coefficient": "uniform", "group": null, "index": 64, "kind": "output", "layer": null, "offset": null, "sources": 9, "use_uniform": true } ], "visits_sha256": "e8eb3dec5a2d1247a9dc230971db785b08d1d64920a8e4a5ecffd27a07d55f4c" } }, "uniform_depth_all": { "bits_per_byte": 1.8740129107275643, "forward": { "activation_sha256": { "attention_branch_output": "5e6ecbcd78543fc42d4098a51169dca446d404ce384015643e84d2c78433c949", "mlp_branch_output": "c0b8018c349b1bd09e5d17858eb312a2323da0e92de9651e2bb08ae4adf115c7", "post_attention_state": "90a633f12d984935028fe5d7212be48e7da7b4a0806751939a551938d16dc540", "post_mlp_state": "89f2226456bfe96e245d2aad0f699d32c676f9f53099789cd870253d002402f5", "pre_attention_input": "ee695e78937de01b7128d3866b8fc5dd328ebdf2c159be37918d4f02952d5248", "pre_mlp_input": "e53fa425d89dfeb0e9395eca9bcd73dc0a55314abf45c5b10b43e2cbfa724345" }, "logits_sha256": "bdacba5675f0579cec629bbf01b36c8660ad576cfdf9fef1bea8259eaeac9ec2", "loss_nats": 1.2989667654037476, "mixer_summary_sha256": "eb3ac0f07de9b755d6e67a510ab68d09d9ad666bb7b5d020ba7d8c5055c399b2" }, "loss_nats": 1.2989667654037476, "loss_scale": 1.0, "mixers": [ { "branch": "attention", "entropy_mean": -0.0, "group": 1, "latest_source_mass": 1.0, "layer": 1, "max_source_mass": 1.0, "mixer_index": 0, "normalized_entropy": 1.0, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 1.0, "median": 1.0, "p05": 1.0, "p95": 1.0 } ], "sources": 1 }, { "branch": "mlp", "entropy_mean": 0.22886337339878082, "group": 1, "latest_source_mass": 0.06350618600845337, "layer": 1, "max_source_mass": 0.9364938139915466, "mixer_index": 1, "normalized_entropy": 0.3301800538435402, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.9364938139915466, "median": 0.9452826976776123, "p05": 0.8725647330284119, "p95": 0.9712384939193726 }, { "label": "current_group_partial", "mean": 0.06350618600845337, "median": 0.0547172985970974, "p05": 0.028761589899659157, "p95": 0.12743529677391052 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.603729784488678, "group": 1, "latest_source_mass": 0.4133155345916748, "layer": 2, "max_source_mass": 0.5866844654083252, "mixer_index": 2, "normalized_entropy": 0.8709979661187783, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.5866844654083252, "median": 0.5734395384788513, "p05": 0.29813364148139954, "p95": 0.8843642473220825 }, { "label": "current_group_partial", "mean": 0.4133155345916748, "median": 0.4265604615211487, "p05": 0.11563576757907867, "p95": 0.7018663287162781 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.3967982828617096, "group": 1, "latest_source_mass": 0.14110636711120605, "layer": 2, "max_source_mass": 0.858893632888794, "mixer_index": 3, "normalized_entropy": 0.5724589149178446, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.858893632888794, "median": 0.8705973029136658, "p05": 0.7599387168884277, "p95": 0.9270997643470764 }, { "label": "current_group_partial", "mean": 0.14110636711120605, "median": 0.12940272688865662, "p05": 0.07290027290582657, "p95": 0.2400612235069275 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6852496862411499, "group": 1, "latest_source_mass": 0.518975019454956, "layer": 3, "max_source_mass": 0.518975019454956, "mixer_index": 4, "normalized_entropy": 0.9886063241108252, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.48102498054504395, "median": 0.48102718591690063, "p05": 0.38786420226097107, "p95": 0.580143928527832 }, { "label": "current_group_partial", "mean": 0.518975019454956, "median": 0.5189728140830994, "p05": 0.41985607147216797, "p95": 0.6121357679367065 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.5890734195709229, "group": 1, "latest_source_mass": 0.3123423159122467, "layer": 3, "max_source_mass": 0.6876576542854309, "mixer_index": 5, "normalized_entropy": 0.8498533011344741, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.6876576542854309, "median": 0.6909762620925903, "p05": 0.48803940415382385, "p95": 0.8900076746940613 }, { "label": "current_group_partial", "mean": 0.3123423159122467, "median": 0.3090237081050873, "p05": 0.10999235510826111, "p95": 0.5119606256484985 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6873323321342468, "group": 1, "latest_source_mass": 0.4831485152244568, "layer": 4, "max_source_mass": 0.5168514251708984, "mixer_index": 6, "normalized_entropy": 0.9916109470127238, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.5168514251708984, "median": 0.5219584703445435, "p05": 0.4251725673675537, "p95": 0.5947866439819336 }, { "label": "current_group_partial", "mean": 0.4831485152244568, "median": 0.47804147005081177, "p05": 0.405213326215744, "p95": 0.5748274326324463 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.487301766872406, "group": 1, "latest_source_mass": 0.2060384750366211, "layer": 4, "max_source_mass": 0.7939615249633789, "mixer_index": 7, "normalized_entropy": 0.7030278424832499, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.7939615249633789, "median": 0.7953404188156128, "p05": 0.6590759754180908, "p95": 0.9329437017440796 }, { "label": "current_group_partial", "mean": 0.2060384750366211, "median": 0.20465964078903198, "p05": 0.06705628335475922, "p95": 0.3409239649772644 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.674697756767273, "group": 2, "latest_source_mass": 0.42952626943588257, "layer": 5, "max_source_mass": 0.5704737305641174, "mixer_index": 8, "normalized_entropy": 0.9733831077870527, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.5704737305641174, "median": 0.5696132779121399, "p05": 0.4695202708244324, "p95": 0.6750621199607849 }, { "label": "completed_group_1", "mean": 0.42952626943588257, "median": 0.4303867816925049, "p05": 0.3249378800392151, "p95": 0.5304797887802124 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.9691112041473389, "group": 2, "latest_source_mass": 0.2681158185005188, "layer": 5, "max_source_mass": 0.5506307482719421, "mixer_index": 9, "normalized_entropy": 0.8821230329784768, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.5506307482719421, "median": 0.543993353843689, "p05": 0.41262108087539673, "p95": 0.7161091566085815 }, { "label": "completed_group_1", "mean": 0.18125343322753906, "median": 0.1794438362121582, "p05": 0.11575750261545181, "p95": 0.25031501054763794 }, { "label": "current_group_partial", "mean": 0.2681158185005188, "median": 0.2664111256599426, "p05": 0.1564716249704361, "p95": 0.3817881643772125 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0745112895965576, "group": 2, "latest_source_mass": 0.412794291973114, "layer": 6, "max_source_mass": 0.412794291973114, "mixer_index": 10, "normalized_entropy": 0.9780623252441762, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.34262996912002563, "median": 0.3412924110889435, "p05": 0.29556700587272644, "p95": 0.39125555753707886 }, { "label": "completed_group_1", "mean": 0.24457570910453796, "median": 0.24382859468460083, "p05": 0.22120657563209534, "p95": 0.2734297513961792 }, { "label": "current_group_partial", "mean": 0.412794291973114, "median": 0.4151499271392822, "p05": 0.37735897302627563, "p95": 0.43749260902404785 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0074045658111572, "group": 2, "latest_source_mass": 0.3775783181190491, "layer": 6, "max_source_mass": 0.4399106502532959, "mixer_index": 11, "normalized_entropy": 0.9169791528842925, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.4399106502532959, "median": 0.44856905937194824, "p05": 0.22063353657722473, "p95": 0.6359403133392334 }, { "label": "completed_group_1", "mean": 0.18251106142997742, "median": 0.18562838435173035, "p05": 0.1196163073182106, "p95": 0.23045453429222107 }, { "label": "current_group_partial", "mean": 0.3775783181190491, "median": 0.3675365447998047, "p05": 0.23817461729049683, "p95": 0.5557451844215393 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.060497522354126, "group": 2, "latest_source_mass": 0.4025024175643921, "layer": 7, "max_source_mass": 0.4025024175643921, "mixer_index": 12, "normalized_entropy": 0.9653064445872968, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.3786153197288513, "median": 0.3907395303249359, "p05": 0.28039106726646423, "p95": 0.4436405599117279 }, { "label": "completed_group_1", "mean": 0.2188822478055954, "median": 0.21226292848587036, "p05": 0.1885872334241867, "p95": 0.2672030031681061 }, { "label": "current_group_partial", "mean": 0.4025024175643921, "median": 0.39815032482147217, "p05": 0.3533596992492676, "p95": 0.460100919008255 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 0.9941019415855408, "group": 2, "latest_source_mass": 0.4801652431488037, "layer": 7, "max_source_mass": 0.4801652431488037, "mixer_index": 13, "normalized_entropy": 0.9048705824970601, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.32296955585479736, "median": 0.2803739905357361, "p05": 0.1784922033548355, "p95": 0.6440245509147644 }, { "label": "completed_group_1", "mean": 0.19686520099639893, "median": 0.2053825855255127, "p05": 0.09813719242811203, "p95": 0.2504696547985077 }, { "label": "current_group_partial", "mean": 0.4801652431488037, "median": 0.5057140588760376, "p05": 0.2529003322124481, "p95": 0.598870575428009 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0503522157669067, "group": 2, "latest_source_mass": 0.4001518487930298, "layer": 8, "max_source_mass": 0.40130966901779175, "mixer_index": 14, "normalized_entropy": 0.9560717885654542, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.40130966901779175, "median": 0.396759569644928, "p05": 0.3479611873626709, "p95": 0.4733521044254303 }, { "label": "completed_group_1", "mean": 0.19853851199150085, "median": 0.19769755005836487, "p05": 0.17136667668819427, "p95": 0.22788912057876587 }, { "label": "current_group_partial", "mean": 0.4001518487930298, "median": 0.40291696786880493, "p05": 0.3506356179714203, "p95": 0.43941178917884827 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0129402875900269, "group": 2, "latest_source_mass": 0.42161279916763306, "layer": 8, "max_source_mass": 0.42161279916763306, "mixer_index": 15, "normalized_entropy": 0.9220179839951123, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.4093267023563385, "median": 0.4046899676322937, "p05": 0.2895161807537079, "p95": 0.5762131810188293 }, { "label": "completed_group_1", "mean": 0.16906049847602844, "median": 0.16888955235481262, "p05": 0.11233551055192947, "p95": 0.22477403283119202 }, { "label": "current_group_partial", "mean": 0.42161279916763306, "median": 0.4227708578109741, "p05": 0.3131049573421478, "p95": 0.519075334072113 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0774019956588745, "group": 3, "latest_source_mass": 0.3560434877872467, "layer": 9, "max_source_mass": 0.38510820269584656, "mixer_index": 16, "normalized_entropy": 0.9806935592947451, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.38510820269584656, "median": 0.3892287313938141, "p05": 0.2934730052947998, "p95": 0.4718902111053467 }, { "label": "completed_group_1", "mean": 0.25884830951690674, "median": 0.25734007358551025, "p05": 0.22217319905757904, "p95": 0.29551801085472107 }, { "label": "completed_group_2", "mean": 0.3560434877872467, "median": 0.35412120819091797, "p05": 0.29444968700408936, "p95": 0.41908568143844604 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.2740377187728882, "group": 3, "latest_source_mass": 0.3633861243724823, "layer": 9, "max_source_mass": 0.3633861243724823, "mixer_index": 17, "normalized_entropy": 0.9190239493895668, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.29550763964653015, "median": 0.30360478162765503, "p05": 0.16904757916927338, "p95": 0.45146965980529785 }, { "label": "completed_group_1", "mean": 0.09541244804859161, "median": 0.09486871212720871, "p05": 0.061360105872154236, "p95": 0.13037073612213135 }, { "label": "completed_group_2", "mean": 0.24569380283355713, "median": 0.24333015084266663, "p05": 0.1698218435049057, "p95": 0.31589236855506897 }, { "label": "current_group_partial", "mean": 0.3633861243724823, "median": 0.36472851037979126, "p05": 0.2795379161834717, "p95": 0.4399876892566681 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.360302448272705, "group": 3, "latest_source_mass": 0.28569501638412476, "layer": 10, "max_source_mass": 0.3109527826309204, "mixer_index": 18, "normalized_entropy": 0.9812507981160736, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.3109527826309204, "median": 0.32172977924346924, "p05": 0.23520725965499878, "p95": 0.37052470445632935 }, { "label": "completed_group_1", "mean": 0.20984819531440735, "median": 0.20958667993545532, "p05": 0.1891367882490158, "p95": 0.23184512555599213 }, { "label": "completed_group_2", "mean": 0.19350400567054749, "median": 0.19153150916099548, "p05": 0.16607457399368286, "p95": 0.22877082228660583 }, { "label": "current_group_partial", "mean": 0.28569501638412476, "median": 0.28060320019721985, "p05": 0.23824527859687805, "p95": 0.3439165949821472 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.2722704410552979, "group": 3, "latest_source_mass": 0.29912811517715454, "layer": 10, "max_source_mass": 0.35036465525627136, "mixer_index": 19, "normalized_entropy": 0.9177491279900463, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.35036465525627136, "median": 0.36210548877716064, "p05": 0.12807878851890564, "p95": 0.5797606706619263 }, { "label": "completed_group_1", "mean": 0.12631990015506744, "median": 0.1264108121395111, "p05": 0.06977735459804535, "p95": 0.1755298376083374 }, { "label": "completed_group_2", "mean": 0.22418735921382904, "median": 0.22563719749450684, "p05": 0.1176326721906662, "p95": 0.32001155614852905 }, { "label": "current_group_partial", "mean": 0.29912811517715454, "median": 0.28589946031570435, "p05": 0.193109393119812, "p95": 0.44119665026664734 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3558036088943481, "group": 3, "latest_source_mass": 0.314197838306427, "layer": 11, "max_source_mass": 0.314197838306427, "mixer_index": 20, "normalized_entropy": 0.978005571485618, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.298976868391037, "median": 0.2961939573287964, "p05": 0.24950163066387177, "p95": 0.35835936665534973 }, { "label": "completed_group_1", "mean": 0.1941717267036438, "median": 0.1918032318353653, "p05": 0.16809208691120148, "p95": 0.22895419597625732 }, { "label": "completed_group_2", "mean": 0.1926535815000534, "median": 0.19306811690330505, "p05": 0.16983944177627563, "p95": 0.2155935913324356 }, { "label": "current_group_partial", "mean": 0.314197838306427, "median": 0.31541913747787476, "p05": 0.2673022747039795, "p95": 0.35941022634506226 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.2696614265441895, "group": 3, "latest_source_mass": 0.2862822115421295, "layer": 11, "max_source_mass": 0.3667730689048767, "mixer_index": 21, "normalized_entropy": 0.9158671218416545, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.3667730689048767, "median": 0.4067659378051758, "p05": 0.10593366622924805, "p95": 0.5672274827957153 }, { "label": "completed_group_1", "mean": 0.15310388803482056, "median": 0.15050218999385834, "p05": 0.08651046454906464, "p95": 0.22459912300109863 }, { "label": "completed_group_2", "mean": 0.19384080171585083, "median": 0.18732509016990662, "p05": 0.1267009675502777, "p95": 0.281107634305954 }, { "label": "current_group_partial", "mean": 0.2862822115421295, "median": 0.26271265745162964, "p05": 0.17024393379688263, "p95": 0.475044310092926 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3606936931610107, "group": 3, "latest_source_mass": 0.3028184771537781, "layer": 12, "max_source_mass": 0.3028184771537781, "mixer_index": 22, "normalized_entropy": 0.9815330216461395, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.298227459192276, "median": 0.2954302132129669, "p05": 0.25287362933158875, "p95": 0.35405808687210083 }, { "label": "completed_group_1", "mean": 0.18611131608486176, "median": 0.1850002110004425, "p05": 0.15901757776737213, "p95": 0.21725836396217346 }, { "label": "completed_group_2", "mean": 0.21284276247024536, "median": 0.21402311325073242, "p05": 0.18819917738437653, "p95": 0.23418807983398438 }, { "label": "current_group_partial", "mean": 0.3028184771537781, "median": 0.3041122555732727, "p05": 0.25661808252334595, "p95": 0.34458184242248535 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.269316554069519, "group": 3, "latest_source_mass": 0.2653314769268036, "layer": 12, "max_source_mass": 0.4081852436065674, "mixer_index": 23, "normalized_entropy": 0.9156183489371815, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.4081852436065674, "median": 0.4254342317581177, "p05": 0.20765496790409088, "p95": 0.587238073348999 }, { "label": "completed_group_1", "mean": 0.14420127868652344, "median": 0.1438048928976059, "p05": 0.1064920499920845, "p95": 0.1871720254421234 }, { "label": "completed_group_2", "mean": 0.18228203058242798, "median": 0.17507661879062653, "p05": 0.12183455377817154, "p95": 0.26505547761917114 }, { "label": "current_group_partial", "mean": 0.2653314769268036, "median": 0.24924281239509583, "p05": 0.15609969198703766, "p95": 0.4061311185359955 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3636136054992676, "group": 4, "latest_source_mass": 0.3136237859725952, "layer": 13, "max_source_mass": 0.3136237859725952, "mixer_index": 24, "normalized_entropy": 0.9836392931712563, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.27427661418914795, "median": 0.27403637766838074, "p05": 0.22928935289382935, "p95": 0.319773405790329 }, { "label": "completed_group_1", "mean": 0.21341991424560547, "median": 0.20697325468063354, "p05": 0.18552586436271667, "p95": 0.26688331365585327 }, { "label": "completed_group_2", "mean": 0.19867971539497375, "median": 0.197545126080513, "p05": 0.1794913411140442, "p95": 0.222100168466568 }, { "label": "completed_group_3", "mean": 0.3136237859725952, "median": 0.31875866651535034, "p05": 0.24156072735786438, "p95": 0.3695686459541321 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.570773959159851, "group": 4, "latest_source_mass": 0.2294243425130844, "layer": 13, "max_source_mass": 0.2658740282058716, "mixer_index": 25, "normalized_entropy": 0.9759767351225285, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.2658740282058716, "median": 0.2747163474559784, "p05": 0.17927880585193634, "p95": 0.3512483835220337 }, { "label": "completed_group_1", "mean": 0.13419361412525177, "median": 0.13374894857406616, "p05": 0.11072897166013718, "p95": 0.1586270034313202 }, { "label": "completed_group_2", "mean": 0.1907704919576645, "median": 0.19074805080890656, "p05": 0.14699944853782654, "p95": 0.23316305875778198 }, { "label": "completed_group_3", "mean": 0.17973749339580536, "median": 0.17616114020347595, "p05": 0.1363743096590042, "p95": 0.23202598094940186 }, { "label": "current_group_partial", "mean": 0.2294243425130844, "median": 0.22995705902576447, "p05": 0.17975632846355438, "p95": 0.2789157032966614 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.5560038089752197, "group": 4, "latest_source_mass": 0.32468360662460327, "layer": 14, "max_source_mass": 0.32468360662460327, "mixer_index": 26, "normalized_entropy": 0.9667995248241249, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.20477107167243958, "median": 0.206403911113739, "p05": 0.16619175672531128, "p95": 0.2431035190820694 }, { "label": "completed_group_1", "mean": 0.15721669793128967, "median": 0.15658551454544067, "p05": 0.13878680765628815, "p95": 0.17778633534908295 }, { "label": "completed_group_2", "mean": 0.13811248540878296, "median": 0.13743364810943604, "p05": 0.12352626770734787, "p95": 0.15553249418735504 }, { "label": "completed_group_3", "mean": 0.1752161681652069, "median": 0.17309701442718506, "p05": 0.15223677456378937, "p95": 0.20474471151828766 }, { "label": "current_group_partial", "mean": 0.32468360662460327, "median": 0.32497870922088623, "p05": 0.2946181893348694, "p95": 0.35203880071640015 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.5659973621368408, "group": 4, "latest_source_mass": 0.26703086495399475, "layer": 14, "max_source_mass": 0.26703086495399475, "mixer_index": 27, "normalized_entropy": 0.9730088685238187, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.20531430840492249, "median": 0.20229148864746094, "p05": 0.10698921978473663, "p95": 0.3393132984638214 }, { "label": "completed_group_1", "mean": 0.15986156463623047, "median": 0.15952220559120178, "p05": 0.11969029903411865, "p95": 0.20180393755435944 }, { "label": "completed_group_2", "mean": 0.21557100117206573, "median": 0.21654027700424194, "p05": 0.1673085242509842, "p95": 0.2610440254211426 }, { "label": "completed_group_3", "mean": 0.15222224593162537, "median": 0.1510905921459198, "p05": 0.11800260841846466, "p95": 0.18893754482269287 }, { "label": "current_group_partial", "mean": 0.26703086495399475, "median": 0.258539617061615, "p05": 0.19330690801143646, "p95": 0.36521556973457336 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.5552496910095215, "group": 4, "latest_source_mass": 0.31774693727493286, "layer": 15, "max_source_mass": 0.31774693727493286, "mixer_index": 28, "normalized_entropy": 0.9663309649872576, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.2162160575389862, "median": 0.2145705670118332, "p05": 0.17727568745613098, "p95": 0.26878347992897034 }, { "label": "completed_group_1", "mean": 0.17982497811317444, "median": 0.18024154007434845, "p05": 0.15453973412513733, "p95": 0.2030555009841919 }, { "label": "completed_group_2", "mean": 0.1307673156261444, "median": 0.13107559084892273, "p05": 0.11463287472724915, "p95": 0.14630235731601715 }, { "label": "completed_group_3", "mean": 0.1554446816444397, "median": 0.15516987442970276, "p05": 0.13627474009990692, "p95": 0.1754152774810791 }, { "label": "current_group_partial", "mean": 0.31774693727493286, "median": 0.319039523601532, "p05": 0.27606821060180664, "p95": 0.35650834441185 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.5068919658660889, "group": 4, "latest_source_mass": 0.3298327624797821, "layer": 15, "max_source_mass": 0.3298327624797821, "mixer_index": 29, "normalized_entropy": 0.9362846209998111, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.20646697282791138, "median": 0.18775638937950134, "p05": 0.0806753933429718, "p95": 0.4274584949016571 }, { "label": "completed_group_1", "mean": 0.1197461187839508, "median": 0.11989086866378784, "p05": 0.0809398666024208, "p95": 0.15743419528007507 }, { "label": "completed_group_2", "mean": 0.19215428829193115, "median": 0.19620640575885773, "p05": 0.1148109883069992, "p95": 0.2491212636232376 }, { "label": "completed_group_3", "mean": 0.15179988741874695, "median": 0.15212899446487427, "p05": 0.10697861760854721, "p95": 0.19542087614536285 }, { "label": "current_group_partial", "mean": 0.3298327624797821, "median": 0.32837340235710144, "p05": 0.22746729850769043, "p95": 0.4339301884174347 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.5435223579406738, "group": 4, "latest_source_mass": 0.3224271535873413, "layer": 16, "max_source_mass": 0.3224271535873413, "mixer_index": 30, "normalized_entropy": 0.9590443632623663, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.22410975396633148, "median": 0.22760415077209473, "p05": 0.14593973755836487, "p95": 0.30093055963516235 }, { "label": "completed_group_1", "mean": 0.16986176371574402, "median": 0.16963240504264832, "p05": 0.13569067418575287, "p95": 0.20651353895664215 }, { "label": "completed_group_2", "mean": 0.13080403208732605, "median": 0.13075244426727295, "p05": 0.10945598781108856, "p95": 0.15187400579452515 }, { "label": "completed_group_3", "mean": 0.15279728174209595, "median": 0.15036612749099731, "p05": 0.12810495495796204, "p95": 0.18468116223812103 }, { "label": "current_group_partial", "mean": 0.3224271535873413, "median": 0.3164973258972168, "p05": 0.2600978910923004, "p95": 0.40530598163604736 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.4949920177459717, "group": 4, "latest_source_mass": 0.31847020983695984, "layer": 16, "max_source_mass": 0.31847020983695984, "mixer_index": 31, "normalized_entropy": 0.9288907675133354, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.27174079418182373, "median": 0.2585451304912567, "p05": 0.13639721274375916, "p95": 0.4698348045349121 }, { "label": "completed_group_1", "mean": 0.12178683280944824, "median": 0.11714950203895569, "p05": 0.08594125509262085, "p95": 0.1723237782716751 }, { "label": "completed_group_2", "mean": 0.1529826819896698, "median": 0.15223106741905212, "p05": 0.10371473431587219, "p95": 0.203575000166893 }, { "label": "completed_group_3", "mean": 0.1350194811820984, "median": 0.13363957405090332, "p05": 0.08964549005031586, "p95": 0.18468207120895386 }, { "label": "current_group_partial", "mean": 0.31847020983695984, "median": 0.3185216188430786, "p05": 0.21444711089134216, "p95": 0.4301546812057495 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.544264793395996, "group": 5, "latest_source_mass": 0.2861037850379944, "layer": 17, "max_source_mass": 0.2861037850379944, "mixer_index": 32, "normalized_entropy": 0.9595056643474137, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.24016115069389343, "median": 0.23868846893310547, "p05": 0.18435582518577576, "p95": 0.2983453571796417 }, { "label": "completed_group_1", "mean": 0.20730282366275787, "median": 0.1998077780008316, "p05": 0.16255050897598267, "p95": 0.28107208013534546 }, { "label": "completed_group_2", "mean": 0.09692573547363281, "median": 0.0967404842376709, "p05": 0.08098706603050232, "p95": 0.11221462488174438 }, { "label": "completed_group_3", "mean": 0.1695065200328827, "median": 0.1696585714817047, "p05": 0.136396586894989, "p95": 0.20130634307861328 }, { "label": "completed_group_4", "mean": 0.2861037850379944, "median": 0.28883200883865356, "p05": 0.22401469945907593, "p95": 0.3409370183944702 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.686682939529419, "group": 5, "latest_source_mass": 0.12281867861747742, "layer": 17, "max_source_mass": 0.2885604202747345, "mixer_index": 33, "normalized_entropy": 0.9413556721740635, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.16755425930023193, "median": 0.1327776312828064, "p05": 0.0790192037820816, "p95": 0.3667412996292114 }, { "label": "completed_group_1", "mean": 0.10962115228176117, "median": 0.10525335371494293, "p05": 0.07840172946453094, "p95": 0.15405339002609253 }, { "label": "completed_group_2", "mean": 0.1783454418182373, "median": 0.17896884679794312, "p05": 0.10802385956048965, "p95": 0.24048911035060883 }, { "label": "completed_group_3", "mean": 0.13310004770755768, "median": 0.13033035397529602, "p05": 0.0904831737279892, "p95": 0.17934636771678925 }, { "label": "completed_group_4", "mean": 0.2885604202747345, "median": 0.2831825315952301, "p05": 0.16967125236988068, "p95": 0.4189499616622925 }, { "label": "current_group_partial", "mean": 0.12281867861747742, "median": 0.12062528729438782, "p05": 0.07461707293987274, "p95": 0.1782219111919403 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.7070834636688232, "group": 5, "latest_source_mass": 0.2988377809524536, "layer": 18, "max_source_mass": 0.2988377809524536, "mixer_index": 34, "normalized_entropy": 0.9527414214834803, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.181177020072937, "median": 0.18419909477233887, "p05": 0.13624237477779388, "p95": 0.22192169725894928 }, { "label": "completed_group_1", "mean": 0.14557164907455444, "median": 0.144476979970932, "p05": 0.1293315887451172, "p95": 0.16486899554729462 }, { "label": "completed_group_2", "mean": 0.08318156749010086, "median": 0.0820261612534523, "p05": 0.07088497281074524, "p95": 0.0990293100476265 }, { "label": "completed_group_3", "mean": 0.11424209177494049, "median": 0.11267061531543732, "p05": 0.0979941189289093, "p95": 0.1353992223739624 }, { "label": "completed_group_4", "mean": 0.17698991298675537, "median": 0.17597860097885132, "p05": 0.14837387204170227, "p95": 0.21050144731998444 }, { "label": "current_group_partial", "mean": 0.2988377809524536, "median": 0.2989804744720459, "p05": 0.2721370458602905, "p95": 0.32572072744369507 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.7419235706329346, "group": 5, "latest_source_mass": 0.26016175746917725, "layer": 18, "max_source_mass": 0.26016175746917725, "mixer_index": 35, "normalized_entropy": 0.972186055410333, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1532188355922699, "median": 0.1451343297958374, "p05": 0.11514341086149216, "p95": 0.22103679180145264 }, { "label": "completed_group_1", "mean": 0.11420352011919022, "median": 0.11261284351348877, "p05": 0.0928458571434021, "p95": 0.14123988151550293 }, { "label": "completed_group_2", "mean": 0.16597424447536469, "median": 0.1655328869819641, "p05": 0.13871800899505615, "p95": 0.19611573219299316 }, { "label": "completed_group_3", "mean": 0.1250365972518921, "median": 0.12369821965694427, "p05": 0.10375828295946121, "p95": 0.14912742376327515 }, { "label": "completed_group_4", "mean": 0.18140508234500885, "median": 0.1796584278345108, "p05": 0.14511024951934814, "p95": 0.22286373376846313 }, { "label": "current_group_partial", "mean": 0.26016175746917725, "median": 0.26266711950302124, "p05": 0.1889389455318451, "p95": 0.3234381675720215 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.7127439975738525, "group": 5, "latest_source_mass": 0.2907913327217102, "layer": 19, "max_source_mass": 0.2907913327217102, "mixer_index": 36, "normalized_entropy": 0.9559006256078307, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.18731768429279327, "median": 0.19042012095451355, "p05": 0.13879702985286713, "p95": 0.23792000114917755 }, { "label": "completed_group_1", "mean": 0.16715484857559204, "median": 0.1667211651802063, "p05": 0.14574480056762695, "p95": 0.19026139378547668 }, { "label": "completed_group_2", "mean": 0.08496320247650146, "median": 0.08452782034873962, "p05": 0.07410907745361328, "p95": 0.097386933863163 }, { "label": "completed_group_3", "mean": 0.11362075805664062, "median": 0.11248823255300522, "p05": 0.0977015346288681, "p95": 0.13219426572322845 }, { "label": "completed_group_4", "mean": 0.15615218877792358, "median": 0.15540224313735962, "p05": 0.1289147436618805, "p95": 0.18598109483718872 }, { "label": "current_group_partial", "mean": 0.2907913327217102, "median": 0.2900415062904358, "p05": 0.25822532176971436, "p95": 0.32475030422210693 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.6076834201812744, "group": 5, "latest_source_mass": 0.3738824725151062, "layer": 19, "max_source_mass": 0.3738824725151062, "mixer_index": 37, "normalized_entropy": 0.8972652009334232, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.15032559633255005, "median": 0.12544110417366028, "p05": 0.06976518034934998, "p95": 0.35840943455696106 }, { "label": "completed_group_1", "mean": 0.08624927699565887, "median": 0.08602266013622284, "p05": 0.061998721212148666, "p95": 0.11138717085123062 }, { "label": "completed_group_2", "mean": 0.15736976265907288, "median": 0.15802210569381714, "p05": 0.10121746361255646, "p95": 0.20687000453472137 }, { "label": "completed_group_3", "mean": 0.08584291487932205, "median": 0.0827617198228836, "p05": 0.060561805963516235, "p95": 0.11351694166660309 }, { "label": "completed_group_4", "mean": 0.14632999897003174, "median": 0.1426895558834076, "p05": 0.10106660425662994, "p95": 0.20179018378257751 }, { "label": "current_group_partial", "mean": 0.3738824725151062, "median": 0.3799295425415039, "p05": 0.23631010949611664, "p95": 0.4884527325630188 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.709290862083435, "group": 5, "latest_source_mass": 0.2901708781719208, "layer": 20, "max_source_mass": 0.2901708781719208, "mixer_index": 38, "normalized_entropy": 0.9539733939957076, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.18991519510746002, "median": 0.1924753338098526, "p05": 0.14201581478118896, "p95": 0.23709386587142944 }, { "label": "completed_group_1", "mean": 0.17091524600982666, "median": 0.17006167769432068, "p05": 0.15297070145606995, "p95": 0.19153237342834473 }, { "label": "completed_group_2", "mean": 0.07822999358177185, "median": 0.07778562605381012, "p05": 0.06707640737295151, "p95": 0.09089364111423492 }, { "label": "completed_group_3", "mean": 0.116765096783638, "median": 0.11516137421131134, "p05": 0.10162825882434845, "p95": 0.13653048872947693 }, { "label": "completed_group_4", "mean": 0.1540035903453827, "median": 0.15287795662879944, "p05": 0.12593276798725128, "p95": 0.18590238690376282 }, { "label": "current_group_partial", "mean": 0.2901708781719208, "median": 0.29104921221733093, "p05": 0.25643545389175415, "p95": 0.3205650746822357 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.6690887212753296, "group": 5, "latest_source_mass": 0.3342471718788147, "layer": 20, "max_source_mass": 0.3342471718788147, "mixer_index": 39, "normalized_entropy": 0.9315361520005944, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.19029226899147034, "median": 0.19644704461097717, "p05": 0.09456793963909149, "p95": 0.30468082427978516 }, { "label": "completed_group_1", "mean": 0.1100241169333458, "median": 0.10808776319026947, "p05": 0.09035106748342514, "p95": 0.1359245330095291 }, { "label": "completed_group_2", "mean": 0.11471512168645859, "median": 0.11494918912649155, "p05": 0.08799737691879272, "p95": 0.14024731516838074 }, { "label": "completed_group_3", "mean": 0.09999909996986389, "median": 0.09682698547840118, "p05": 0.07921088486909866, "p95": 0.1264391839504242 }, { "label": "completed_group_4", "mean": 0.1507222205400467, "median": 0.1476232260465622, "p05": 0.11397356539964676, "p95": 0.19991227984428406 }, { "label": "current_group_partial", "mean": 0.3342471718788147, "median": 0.33802270889282227, "p05": 0.2548266053199768, "p95": 0.40130624175071716 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.7458693981170654, "group": 6, "latest_source_mass": 0.2361394464969635, "layer": 21, "max_source_mass": 0.2361394464969635, "mixer_index": 40, "normalized_entropy": 0.9743882636597644, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.20528709888458252, "median": 0.2140858769416809, "p05": 0.13168928027153015, "p95": 0.27098023891448975 }, { "label": "completed_group_1", "mean": 0.1722964197397232, "median": 0.17238952219486237, "p05": 0.15053534507751465, "p95": 0.19365043938159943 }, { "label": "completed_group_2", "mean": 0.10435444861650467, "median": 0.10361101478338242, "p05": 0.08888213336467743, "p95": 0.12212536484003067 }, { "label": "completed_group_3", "mean": 0.12075909972190857, "median": 0.1180175170302391, "p05": 0.10358668118715286, "p95": 0.14458929002285004 }, { "label": "completed_group_4", "mean": 0.16116347908973694, "median": 0.15841388702392578, "p05": 0.1337081342935562, "p95": 0.19791515171527863 }, { "label": "completed_group_5", "mean": 0.2361394464969635, "median": 0.23709020018577576, "p05": 0.20446214079856873, "p95": 0.26509594917297363 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.4612125158309937, "group": 6, "latest_source_mass": 0.5543455481529236, "layer": 21, "max_source_mass": 0.5543455481529236, "mixer_index": 41, "normalized_entropy": 0.7509146897354808, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.08934466540813446, "median": 0.07911523431539536, "p05": 0.04488275945186615, "p95": 0.1737232506275177 }, { "label": "completed_group_1", "mean": 0.054379627108573914, "median": 0.05344422906637192, "p05": 0.04221750423312187, "p95": 0.06924191862344742 }, { "label": "completed_group_2", "mean": 0.07567165046930313, "median": 0.07438549399375916, "p05": 0.05811355262994766, "p95": 0.09632256627082825 }, { "label": "completed_group_3", "mean": 0.05758609622716904, "median": 0.05612001568078995, "p05": 0.044792283326387405, "p95": 0.07347827404737473 }, { "label": "completed_group_4", "mean": 0.07884950935840607, "median": 0.07757881283760071, "p05": 0.05758974328637123, "p95": 0.10452556610107422 }, { "label": "completed_group_5", "mean": 0.08982288837432861, "median": 0.08912864327430725, "p05": 0.0663854256272316, "p95": 0.11604885756969452 }, { "label": "current_group_partial", "mean": 0.5543455481529236, "median": 0.5596089363098145, "p05": 0.45455610752105713, "p95": 0.6367484331130981 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.7241814136505127, "group": 6, "latest_source_mass": 0.39484071731567383, "layer": 22, "max_source_mass": 0.39484071731567383, "mixer_index": 42, "normalized_entropy": 0.886053970419732, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.13360010087490082, "median": 0.14072942733764648, "p05": 0.07699814438819885, "p95": 0.17506597936153412 }, { "label": "completed_group_1", "mean": 0.10609637200832367, "median": 0.10586297512054443, "p05": 0.08648421615362167, "p95": 0.12602579593658447 }, { "label": "completed_group_2", "mean": 0.06363911926746368, "median": 0.06324908882379532, "p05": 0.054980404675006866, "p95": 0.07316673547029495 }, { "label": "completed_group_3", "mean": 0.07426095008850098, "median": 0.07362888753414154, "p05": 0.06318790465593338, "p95": 0.08665015548467636 }, { "label": "completed_group_4", "mean": 0.09383076429367065, "median": 0.09294568747282028, "p05": 0.07638302445411682, "p95": 0.11445100605487823 }, { "label": "completed_group_5", "mean": 0.13373197615146637, "median": 0.13327741622924805, "p05": 0.11399447172880173, "p95": 0.15641780197620392 }, { "label": "current_group_partial", "mean": 0.39484071731567383, "median": 0.3879151940345764, "p05": 0.3302172124385834, "p95": 0.4852263331413269 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.3979300260543823, "group": 6, "latest_source_mass": 0.5830056071281433, "layer": 22, "max_source_mass": 0.5830056071281433, "mixer_index": 43, "normalized_entropy": 0.7183939231382495, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.08959861099720001, "median": 0.07544538378715515, "p05": 0.04165133833885193, "p95": 0.20099139213562012 }, { "label": "completed_group_1", "mean": 0.05414731800556183, "median": 0.05336771905422211, "p05": 0.04069708660244942, "p95": 0.07046990841627121 }, { "label": "completed_group_2", "mean": 0.07668881118297577, "median": 0.07607124000787735, "p05": 0.0555989071726799, "p95": 0.09980960935354233 }, { "label": "completed_group_3", "mean": 0.05603286623954773, "median": 0.054573021829128265, "p05": 0.04162490367889404, "p95": 0.0730118378996849 }, { "label": "completed_group_4", "mean": 0.06979228556156158, "median": 0.06884932518005371, "p05": 0.04576495662331581, "p95": 0.09806846082210541 }, { "label": "completed_group_5", "mean": 0.07073447108268738, "median": 0.07023533433675766, "p05": 0.04956750199198723, "p95": 0.09379896521568298 }, { "label": "current_group_partial", "mean": 0.5830056071281433, "median": 0.5862616896629333, "p05": 0.4658474922180176, "p95": 0.6849814057350159 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.7459560632705688, "group": 6, "latest_source_mass": 0.37691548466682434, "layer": 23, "max_source_mass": 0.37691548466682434, "mixer_index": 44, "normalized_entropy": 0.897243926765161, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.14318783581256866, "median": 0.1493408977985382, "p05": 0.08292607218027115, "p95": 0.19464610517024994 }, { "label": "completed_group_1", "mean": 0.1094653457403183, "median": 0.10858799517154694, "p05": 0.09432064741849899, "p95": 0.1279931515455246 }, { "label": "completed_group_2", "mean": 0.06349228322505951, "median": 0.06325754523277283, "p05": 0.05439673364162445, "p95": 0.0737837627530098 }, { "label": "completed_group_3", "mean": 0.07859592139720917, "median": 0.0775163471698761, "p05": 0.06603655964136124, "p95": 0.09357087314128876 }, { "label": "completed_group_4", "mean": 0.09567482024431229, "median": 0.09468592703342438, "p05": 0.07671141624450684, "p95": 0.11774884164333344 }, { "label": "completed_group_5", "mean": 0.13266831636428833, "median": 0.13199329376220703, "p05": 0.11244359612464905, "p95": 0.15504513680934906 }, { "label": "current_group_partial", "mean": 0.37691548466682434, "median": 0.37180638313293457, "p05": 0.31207239627838135, "p95": 0.4595233201980591 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.4055393934249878, "group": 6, "latest_source_mass": 0.5772827863693237, "layer": 23, "max_source_mass": 0.5772827863693237, "mixer_index": 45, "normalized_entropy": 0.7223043644164862, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.10154955089092255, "median": 0.08766824007034302, "p05": 0.04901552200317383, "p95": 0.20287738740444183 }, { "label": "completed_group_1", "mean": 0.05619276314973831, "median": 0.055477917194366455, "p05": 0.041608311235904694, "p95": 0.07371112704277039 }, { "label": "completed_group_2", "mean": 0.07791627198457718, "median": 0.07736755907535553, "p05": 0.055920060724020004, "p95": 0.10158214718103409 }, { "label": "completed_group_3", "mean": 0.053915880620479584, "median": 0.05215863138437271, "p05": 0.03798118978738785, "p95": 0.07156119495630264 }, { "label": "completed_group_4", "mean": 0.06398355215787888, "median": 0.06229078769683838, "p05": 0.040616974234580994, "p95": 0.09270931780338287 }, { "label": "completed_group_5", "mean": 0.06915920227766037, "median": 0.0684593990445137, "p05": 0.046986356377601624, "p95": 0.09408729523420334 }, { "label": "current_group_partial", "mean": 0.5772827863693237, "median": 0.5824689269065857, "p05": 0.4518115520477295, "p95": 0.6926202178001404 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.7853620052337646, "group": 6, "latest_source_mass": 0.3407441973686218, "layer": 24, "max_source_mass": 0.3407441973686218, "mixer_index": 46, "normalized_entropy": 0.9174945750195659, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.15284113585948944, "median": 0.1593906283378601, "p05": 0.08678612112998962, "p95": 0.211342453956604 }, { "label": "completed_group_1", "mean": 0.12206703424453735, "median": 0.12153112143278122, "p05": 0.1060115247964859, "p95": 0.1399766057729721 }, { "label": "completed_group_2", "mean": 0.06508444249629974, "median": 0.06482555717229843, "p05": 0.055350616574287415, "p95": 0.07557765394449234 }, { "label": "completed_group_3", "mean": 0.08347313106060028, "median": 0.08190013468265533, "p05": 0.06966532766819, "p95": 0.10050469636917114 }, { "label": "completed_group_4", "mean": 0.09811970591545105, "median": 0.09664031863212585, "p05": 0.07688035070896149, "p95": 0.12369741499423981 }, { "label": "completed_group_5", "mean": 0.13767032325267792, "median": 0.1373325139284134, "p05": 0.11642646044492722, "p95": 0.1606694608926773 }, { "label": "current_group_partial", "mean": 0.3407441973686218, "median": 0.33805108070373535, "p05": 0.271301805973053, "p95": 0.41781002283096313 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.509842872619629, "group": 6, "latest_source_mass": 0.5227814316749573, "layer": 24, "max_source_mass": 0.5227814316749573, "mixer_index": 47, "normalized_entropy": 0.7759057494780099, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.11788781732320786, "median": 0.10097194463014603, "p05": 0.06265684217214584, "p95": 0.2408325970172882 }, { "label": "completed_group_1", "mean": 0.06822571158409119, "median": 0.0673636943101883, "p05": 0.05209728330373764, "p95": 0.08648915588855743 }, { "label": "completed_group_2", "mean": 0.08556114137172699, "median": 0.08463940769433975, "p05": 0.06234566867351532, "p95": 0.1125786080956459 }, { "label": "completed_group_3", "mean": 0.06067090481519699, "median": 0.058974385261535645, "p05": 0.043267179280519485, "p95": 0.07997901737689972 }, { "label": "completed_group_4", "mean": 0.07228855788707733, "median": 0.06972275674343109, "p05": 0.04689513146877289, "p95": 0.106512151658535 }, { "label": "completed_group_5", "mean": 0.07258446514606476, "median": 0.07082539796829224, "p05": 0.050949547439813614, "p95": 0.09970614314079285 }, { "label": "current_group_partial", "mean": 0.5227814316749573, "median": 0.5290273427963257, "p05": 0.38773614168167114, "p95": 0.6392589807510376 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.839367389678955, "group": 7, "latest_source_mass": 0.29583215713500977, "layer": 25, "max_source_mass": 0.29583215713500977, "mixer_index": 48, "normalized_entropy": 0.9452478525649903, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.15906387567520142, "median": 0.15846745669841766, "p05": 0.09519976377487183, "p95": 0.24517810344696045 }, { "label": "completed_group_1", "mean": 0.13139380514621735, "median": 0.1308426856994629, "p05": 0.11478274315595627, "p95": 0.15000829100608826 }, { "label": "completed_group_2", "mean": 0.07717745751142502, "median": 0.07715500891208649, "p05": 0.06616044044494629, "p95": 0.08791667222976685 }, { "label": "completed_group_3", "mean": 0.09607303142547607, "median": 0.09436552226543427, "p05": 0.08152313530445099, "p95": 0.11384639143943787 }, { "label": "completed_group_4", "mean": 0.1107850894331932, "median": 0.1089654266834259, "p05": 0.08814431726932526, "p95": 0.139452263712883 }, { "label": "completed_group_5", "mean": 0.12967455387115479, "median": 0.1299237608909607, "p05": 0.10832332074642181, "p95": 0.15138982236385345 }, { "label": "completed_group_6", "mean": 0.29583215713500977, "median": 0.29647016525268555, "p05": 0.2276897132396698, "p95": 0.3604946434497833 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.5841834545135498, "group": 7, "latest_source_mass": 0.40767741203308105, "layer": 25, "max_source_mass": 0.40767741203308105, "mixer_index": 49, "normalized_entropy": 0.7618312045616817, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.07896879315376282, "median": 0.06809917092323303, "p05": 0.025932887569069862, "p95": 0.19368338584899902 }, { "label": "completed_group_1", "mean": 0.048752427101135254, "median": 0.04767637699842453, "p05": 0.034443892538547516, "p95": 0.0667153000831604 }, { "label": "completed_group_2", "mean": 0.055707164108753204, "median": 0.0549345538020134, "p05": 0.038696639239788055, "p95": 0.0750277042388916 }, { "label": "completed_group_3", "mean": 0.04139826446771622, "median": 0.03878612071275711, "p05": 0.026673054322600365, "p95": 0.05746251344680786 }, { "label": "completed_group_4", "mean": 0.04449018836021423, "median": 0.04196460545063019, "p05": 0.026162121444940567, "p95": 0.07164917886257172 }, { "label": "completed_group_5", "mean": 0.04285799711942673, "median": 0.04109475016593933, "p05": 0.029393533244729042, "p95": 0.06277710199356079 }, { "label": "completed_group_6", "mean": 0.2801477909088135, "median": 0.2763082981109619, "p05": 0.1455708146095276, "p95": 0.4254714846611023 }, { "label": "current_group_partial", "mean": 0.40767741203308105, "median": 0.4120897948741913, "p05": 0.21399420499801636, "p95": 0.5853031277656555 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.8125314712524414, "group": 7, "latest_source_mass": 0.3700707256793976, "layer": 26, "max_source_mass": 0.3700707256793976, "mixer_index": 50, "normalized_entropy": 0.8716433883436914, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.10889513790607452, "median": 0.11333495378494263, "p05": 0.04850132763385773, "p95": 0.1620434820652008 }, { "label": "completed_group_1", "mean": 0.08656036853790283, "median": 0.08640051633119583, "p05": 0.0650845319032669, "p95": 0.1084962785243988 }, { "label": "completed_group_2", "mean": 0.048123255372047424, "median": 0.04800213500857353, "p05": 0.03576180338859558, "p95": 0.06107594817876816 }, { "label": "completed_group_3", "mean": 0.06561170518398285, "median": 0.06379580497741699, "p05": 0.050968848168849945, "p95": 0.08332888036966324 }, { "label": "completed_group_4", "mean": 0.06917259097099304, "median": 0.06724178791046143, "p05": 0.050207968801259995, "p95": 0.09373342245817184 }, { "label": "completed_group_5", "mean": 0.08286689221858978, "median": 0.08183546364307404, "p05": 0.06535022705793381, "p95": 0.10340933501720428 }, { "label": "completed_group_6", "mean": 0.16869932413101196, "median": 0.16509854793548584, "p05": 0.11059704422950745, "p95": 0.23654375970363617 }, { "label": "current_group_partial", "mean": 0.3700707256793976, "median": 0.3694133758544922, "p05": 0.23441238701343536, "p95": 0.5025027990341187 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.50089693069458, "group": 7, "latest_source_mass": 0.49096518754959106, "layer": 26, "max_source_mass": 0.49096518754959106, "mixer_index": 51, "normalized_entropy": 0.7217788529328457, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.07974998652935028, "median": 0.07402504235506058, "p05": 0.04579866677522659, "p95": 0.1359037309885025 }, { "label": "completed_group_1", "mean": 0.04166398197412491, "median": 0.04038450866937637, "p05": 0.027992289513349533, "p95": 0.05969971418380737 }, { "label": "completed_group_2", "mean": 0.04607272148132324, "median": 0.044438350945711136, "p05": 0.030038591474294662, "p95": 0.06808309257030487 }, { "label": "completed_group_3", "mean": 0.03586764633655548, "median": 0.03317485377192497, "p05": 0.02203454077243805, "p95": 0.05572832003235817 }, { "label": "completed_group_4", "mean": 0.04223340004682541, "median": 0.03930673003196716, "p05": 0.02465122565627098, "p95": 0.06968094408512115 }, { "label": "completed_group_5", "mean": 0.04095568507909775, "median": 0.03919431194663048, "p05": 0.026239857077598572, "p95": 0.06058088690042496 }, { "label": "completed_group_6", "mean": 0.22249138355255127, "median": 0.2125401496887207, "p05": 0.10647299885749817, "p95": 0.3711881935596466 }, { "label": "current_group_partial", "mean": 0.49096518754959106, "median": 0.4988659620285034, "p05": 0.2878011167049408, "p95": 0.6676807403564453 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.8059929609298706, "group": 7, "latest_source_mass": 0.37139976024627686, "layer": 27, "max_source_mass": 0.37139976024627686, "mixer_index": 52, "normalized_entropy": 0.8684990295379666, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.10711923241615295, "median": 0.10972198843955994, "p05": 0.04896187782287598, "p95": 0.16201043128967285 }, { "label": "completed_group_1", "mean": 0.07651707530021667, "median": 0.07666032761335373, "p05": 0.05141128972172737, "p95": 0.10152198374271393 }, { "label": "completed_group_2", "mean": 0.058284737169742584, "median": 0.05812416970729828, "p05": 0.039273619651794434, "p95": 0.07745087146759033 }, { "label": "completed_group_3", "mean": 0.06609587371349335, "median": 0.06383530795574188, "p05": 0.04481930285692215, "p95": 0.09336265176534653 }, { "label": "completed_group_4", "mean": 0.07139429450035095, "median": 0.0694514587521553, "p05": 0.0485839769244194, "p95": 0.10034126043319702 }, { "label": "completed_group_5", "mean": 0.07493151724338531, "median": 0.07319007813930511, "p05": 0.051876723766326904, "p95": 0.10402432829141617 }, { "label": "completed_group_6", "mean": 0.17425751686096191, "median": 0.16937679052352905, "p05": 0.10449051856994629, "p95": 0.2624811828136444 }, { "label": "current_group_partial", "mean": 0.37139976024627686, "median": 0.3732905089855194, "p05": 0.23830299079418182, "p95": 0.5014542937278748 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.5622732639312744, "group": 7, "latest_source_mass": 0.4768889844417572, "layer": 27, "max_source_mass": 0.4768889844417572, "mixer_index": 53, "normalized_entropy": 0.7512946301290215, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.08775827288627625, "median": 0.08504067361354828, "p05": 0.0428270623087883, "p95": 0.144285649061203 }, { "label": "completed_group_1", "mean": 0.047765426337718964, "median": 0.04646281152963638, "p05": 0.031814731657505035, "p95": 0.06807165592908859 }, { "label": "completed_group_2", "mean": 0.053928617388010025, "median": 0.052262187004089355, "p05": 0.035437777638435364, "p95": 0.0784272700548172 }, { "label": "completed_group_3", "mean": 0.04293380305171013, "median": 0.03996016085147858, "p05": 0.02556057646870613, "p95": 0.06623870134353638 }, { "label": "completed_group_4", "mean": 0.04174989461898804, "median": 0.03903091698884964, "p05": 0.025013400241732597, "p95": 0.06869364529848099 }, { "label": "completed_group_5", "mean": 0.04734005779027939, "median": 0.04609068110585213, "p05": 0.029884062707424164, "p95": 0.06911227852106094 }, { "label": "completed_group_6", "mean": 0.20163494348526, "median": 0.19222964346408844, "p05": 0.09963390231132507, "p95": 0.3307305574417114 }, { "label": "current_group_partial", "mean": 0.4768889844417572, "median": 0.48383551836013794, "p05": 0.276212215423584, "p95": 0.6487575173377991 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.8757455348968506, "group": 7, "latest_source_mass": 0.3109493851661682, "layer": 28, "max_source_mass": 0.3109493851661682, "mixer_index": 54, "normalized_entropy": 0.9020429270551009, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.11563669890165329, "median": 0.12087491154670715, "p05": 0.05733206868171692, "p95": 0.1688953936100006 }, { "label": "completed_group_1", "mean": 0.08108697831630707, "median": 0.08185897022485733, "p05": 0.05175650864839554, "p95": 0.10753051191568375 }, { "label": "completed_group_2", "mean": 0.07280858606100082, "median": 0.07357072830200195, "p05": 0.04870978742837906, "p95": 0.0948689877986908 }, { "label": "completed_group_3", "mean": 0.07553227245807648, "median": 0.07362619787454605, "p05": 0.056317005306482315, "p95": 0.09756095707416534 }, { "label": "completed_group_4", "mean": 0.07606014609336853, "median": 0.07456053793430328, "p05": 0.05528895929455757, "p95": 0.1022825762629509 }, { "label": "completed_group_5", "mean": 0.06935927271842957, "median": 0.06843917071819305, "p05": 0.04978285729885101, "p95": 0.09167985618114471 }, { "label": "completed_group_6", "mean": 0.19856664538383484, "median": 0.19508783519268036, "p05": 0.1281973123550415, "p95": 0.27796900272369385 }, { "label": "current_group_partial", "mean": 0.3109493851661682, "median": 0.3105033040046692, "p05": 0.18172834813594818, "p95": 0.43916669487953186 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.6924852132797241, "group": 7, "latest_source_mass": 0.3315581679344177, "layer": 28, "max_source_mass": 0.3315581679344177, "mixer_index": 55, "normalized_entropy": 0.8139133413255193, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.08638577908277512, "median": 0.058608733117580414, "p05": 0.026544082909822464, "p95": 0.2740764319896698 }, { "label": "completed_group_1", "mean": 0.059912845492362976, "median": 0.05901291221380234, "p05": 0.03629481792449951, "p95": 0.08707985281944275 }, { "label": "completed_group_2", "mean": 0.06609142571687698, "median": 0.0647939145565033, "p05": 0.038691695779561996, "p95": 0.09703366458415985 }, { "label": "completed_group_3", "mean": 0.056745611131191254, "median": 0.05418796092271805, "p05": 0.032174739986658096, "p95": 0.08370660990476608 }, { "label": "completed_group_4", "mean": 0.06277773529291153, "median": 0.0600711889564991, "p05": 0.03501531109213829, "p95": 0.10094858705997467 }, { "label": "completed_group_5", "mean": 0.05352145433425903, "median": 0.05179818347096443, "p05": 0.032271772623062134, "p95": 0.08084158599376678 }, { "label": "completed_group_6", "mean": 0.2830069661140442, "median": 0.2759752869606018, "p05": 0.12981083989143372, "p95": 0.46204638481140137 }, { "label": "current_group_partial", "mean": 0.3315581679344177, "median": 0.3204721510410309, "p05": 0.13446375727653503, "p95": 0.5667697787284851 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.8980522155761719, "group": 8, "latest_source_mass": 0.20123262703418732, "layer": 29, "max_source_mass": 0.26602888107299805, "mixer_index": 56, "normalized_entropy": 0.9127701729200177, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.11603894829750061, "median": 0.09991352260112762, "p05": 0.030035395175218582, "p95": 0.2941743731498718 }, { "label": "completed_group_1", "mean": 0.08804173022508621, "median": 0.08880250155925751, "p05": 0.05664174258708954, "p95": 0.11779069900512695 }, { "label": "completed_group_2", "mean": 0.07975024729967117, "median": 0.07984748482704163, "p05": 0.05428915470838547, "p95": 0.10499341040849686 }, { "label": "completed_group_3", "mean": 0.09027000516653061, "median": 0.08898856490850449, "p05": 0.06189651042222977, "p95": 0.11996553838253021 }, { "label": "completed_group_4", "mean": 0.08430293202400208, "median": 0.0819794088602066, "p05": 0.05464346706867218, "p95": 0.12159879505634308 }, { "label": "completed_group_5", "mean": 0.07433462142944336, "median": 0.07287181913852692, "p05": 0.052520088851451874, "p95": 0.1010315865278244 }, { "label": "completed_group_6", "mean": 0.26602888107299805, "median": 0.2639757990837097, "p05": 0.15474572777748108, "p95": 0.3846370279788971 }, { "label": "completed_group_7", "mean": 0.20123262703418732, "median": 0.18140269815921783, "p05": 0.08419936150312424, "p95": 0.3782624900341034 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.7512003183364868, "group": 8, "latest_source_mass": 0.33482813835144043, "layer": 29, "max_source_mass": 0.33482813835144043, "mixer_index": 57, "normalized_entropy": 0.7970056117156376, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.07744461297988892, "median": 0.06970229744911194, "p05": 0.014401955530047417, "p95": 0.17144526541233063 }, { "label": "completed_group_1", "mean": 0.046086348593235016, "median": 0.04394831880927086, "p05": 0.028066720813512802, "p95": 0.0718703418970108 }, { "label": "completed_group_2", "mean": 0.05025433003902435, "median": 0.04861236736178398, "p05": 0.029644077643752098, "p95": 0.07761462777853012 }, { "label": "completed_group_3", "mean": 0.03748488053679466, "median": 0.03434944897890091, "p05": 0.020774798467755318, "p95": 0.05938909202814102 }, { "label": "completed_group_4", "mean": 0.04005061462521553, "median": 0.037498839199543, "p05": 0.019541850313544273, "p95": 0.06922487914562225 }, { "label": "completed_group_5", "mean": 0.041623204946517944, "median": 0.039552345871925354, "p05": 0.022081367671489716, "p95": 0.06821337342262268 }, { "label": "completed_group_6", "mean": 0.198423832654953, "median": 0.18318486213684082, "p05": 0.07548617571592331, "p95": 0.3721975088119507 }, { "label": "completed_group_7", "mean": 0.17380405962467194, "median": 0.1514919549226761, "p05": 0.05777408182621002, "p95": 0.36326974630355835 }, { "label": "current_group_partial", "mean": 0.33482813835144043, "median": 0.33602219820022583, "p05": 0.0795319601893425, "p95": 0.5979493856430054 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.0160694122314453, "group": 8, "latest_source_mass": 0.21711516380310059, "layer": 30, "max_source_mass": 0.21711516380310059, "mixer_index": 58, "normalized_entropy": 0.9175527313077866, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.09385271370410919, "median": 0.09211117774248123, "p05": 0.032377734780311584, "p95": 0.16496515274047852 }, { "label": "completed_group_1", "mean": 0.06208505108952522, "median": 0.06140740588307381, "p05": 0.0407925546169281, "p95": 0.0863042101264 }, { "label": "completed_group_2", "mean": 0.06560768187046051, "median": 0.06516212224960327, "p05": 0.041897669434547424, "p95": 0.09085705131292343 }, { "label": "completed_group_3", "mean": 0.07029052078723907, "median": 0.06780634075403214, "p05": 0.04846132919192314, "p95": 0.09917552024126053 }, { "label": "completed_group_4", "mean": 0.06935097277164459, "median": 0.06708167493343353, "p05": 0.04668639227747917, "p95": 0.09815650433301926 }, { "label": "completed_group_5", "mean": 0.056832700967788696, "median": 0.05595892667770386, "p05": 0.038822539150714874, "p95": 0.0771557092666626 }, { "label": "completed_group_6", "mean": 0.19425815343856812, "median": 0.19145764410495758, "p05": 0.12249615788459778, "p95": 0.27496030926704407 }, { "label": "completed_group_7", "mean": 0.17060703039169312, "median": 0.16317546367645264, "p05": 0.09239593893289566, "p95": 0.2726203501224518 }, { "label": "current_group_partial", "mean": 0.21711516380310059, "median": 0.21198657155036926, "p05": 0.11019522696733475, "p95": 0.3391069769859314 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 1.7837426662445068, "group": 8, "latest_source_mass": 0.29026859998703003, "layer": 30, "max_source_mass": 0.29026859998703003, "mixer_index": 59, "normalized_entropy": 0.8118162725118464, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.08836327493190765, "median": 0.07904411852359772, "p05": 0.02631576545536518, "p95": 0.17534467577934265 }, { "label": "completed_group_1", "mean": 0.04953558370471001, "median": 0.04658038914203644, "p05": 0.025181813165545464, "p95": 0.08318150043487549 }, { "label": "completed_group_2", "mean": 0.04813014343380928, "median": 0.045740917325019836, "p05": 0.026714935898780823, "p95": 0.07843323051929474 }, { "label": "completed_group_3", "mean": 0.0373263955116272, "median": 0.03401809185743332, "p05": 0.01990434154868126, "p95": 0.06290259957313538 }, { "label": "completed_group_4", "mean": 0.037015400826931, "median": 0.03405017405748367, "p05": 0.018646355718374252, "p95": 0.06544196605682373 }, { "label": "completed_group_5", "mean": 0.03390160948038101, "median": 0.03198609501123428, "p05": 0.018189728260040283, "p95": 0.05570102855563164 }, { "label": "completed_group_6", "mean": 0.1840541511774063, "median": 0.17327937483787537, "p05": 0.07966600358486176, "p95": 0.32982927560806274 }, { "label": "completed_group_7", "mean": 0.23140482604503632, "median": 0.22406929731369019, "p05": 0.08831030130386353, "p95": 0.40350067615509033 }, { "label": "current_group_partial", "mean": 0.29026859998703003, "median": 0.2745037376880646, "p05": 0.10280418395996094, "p95": 0.5399085879325867 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 1.9973840713500977, "group": 8, "latest_source_mass": 0.19101998209953308, "layer": 31, "max_source_mass": 0.20831164717674255, "mixer_index": 60, "normalized_entropy": 0.9090486661912383, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.10261696577072144, "median": 0.10389091819524765, "p05": 0.01371697522699833, "p95": 0.21163305640220642 }, { "label": "completed_group_1", "mean": 0.06039813905954361, "median": 0.06085003539919853, "p05": 0.026107093319296837, "p95": 0.09348798543214798 }, { "label": "completed_group_2", "mean": 0.06671924144029617, "median": 0.06656201928853989, "p05": 0.034684643149375916, "p95": 0.09819839894771576 }, { "label": "completed_group_3", "mean": 0.08377155661582947, "median": 0.0804532840847969, "p05": 0.05448959022760391, "p95": 0.12039553374052048 }, { "label": "completed_group_4", "mean": 0.07002593576908112, "median": 0.06737451255321503, "p05": 0.044841229915618896, "p95": 0.10655929893255234 }, { "label": "completed_group_5", "mean": 0.05768897384405136, "median": 0.056077782064676285, "p05": 0.039495617151260376, "p95": 0.08242568373680115 }, { "label": "completed_group_6", "mean": 0.20831164717674255, "median": 0.2021285891532898, "p05": 0.11652229726314545, "p95": 0.31706833839416504 }, { "label": "completed_group_7", "mean": 0.1594475358724594, "median": 0.1546279788017273, "p05": 0.06593266129493713, "p95": 0.2735573351383209 }, { "label": "current_group_partial", "mean": 0.19101998209953308, "median": 0.1713225245475769, "p05": 0.06408411264419556, "p95": 0.3850628137588501 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 1.969987154006958, "group": 8, "latest_source_mass": 0.16864418983459473, "layer": 31, "max_source_mass": 0.24503237009048462, "mixer_index": 61, "normalized_entropy": 0.8965797917640489, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.08120635151863098, "median": 0.061164144426584244, "p05": 0.0259944386780262, "p95": 0.23622018098831177 }, { "label": "completed_group_1", "mean": 0.07894632965326309, "median": 0.07658770680427551, "p05": 0.0496862456202507, "p95": 0.11596276611089706 }, { "label": "completed_group_2", "mean": 0.08615730702877045, "median": 0.08467075228691101, "p05": 0.050409119576215744, "p95": 0.12811316549777985 }, { "label": "completed_group_3", "mean": 0.07609779387712479, "median": 0.0722871869802475, "p05": 0.04099677503108978, "p95": 0.11950768530368805 }, { "label": "completed_group_4", "mean": 0.058728985488414764, "median": 0.056080836802721024, "p05": 0.03323862701654434, "p95": 0.09344407916069031 }, { "label": "completed_group_5", "mean": 0.05076971277594566, "median": 0.04946017265319824, "p05": 0.03047419898211956, "p95": 0.07563400268554688 }, { "label": "completed_group_6", "mean": 0.24503237009048462, "median": 0.24110376834869385, "p05": 0.10613486170768738, "p95": 0.3991503119468689 }, { "label": "completed_group_7", "mean": 0.1544169783592224, "median": 0.1400495171546936, "p05": 0.06199025735259056, "p95": 0.2995939254760742 }, { "label": "current_group_partial", "mean": 0.16864418983459473, "median": 0.15222924947738647, "p05": 0.0543956533074379, "p95": 0.3329208791255951 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.071648359298706, "group": 8, "latest_source_mass": 0.11520501971244812, "layer": 32, "max_source_mass": 0.20955611765384674, "mixer_index": 62, "normalized_entropy": 0.9428478002054053, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.10569609701633453, "median": 0.08689188957214355, "p05": 0.03173588588833809, "p95": 0.24571751058101654 }, { "label": "completed_group_1", "mean": 0.09922641515731812, "median": 0.09795533865690231, "p05": 0.06407676637172699, "p95": 0.14120742678642273 }, { "label": "completed_group_2", "mean": 0.08180510997772217, "median": 0.080217644572258, "p05": 0.054607540369033813, "p95": 0.11442367732524872 }, { "label": "completed_group_3", "mean": 0.10646799206733704, "median": 0.10510502755641937, "p05": 0.07415887713432312, "p95": 0.14216329157352448 }, { "label": "completed_group_4", "mean": 0.0826798751950264, "median": 0.08022409677505493, "p05": 0.05491747707128525, "p95": 0.11860857903957367 }, { "label": "completed_group_5", "mean": 0.06615597754716873, "median": 0.06485182791948318, "p05": 0.04477996751666069, "p95": 0.09238749742507935 }, { "label": "completed_group_6", "mean": 0.20955611765384674, "median": 0.206287682056427, "p05": 0.129981130361557, "p95": 0.2996031939983368 }, { "label": "completed_group_7", "mean": 0.13320739567279816, "median": 0.12465103715658188, "p05": 0.06303662061691284, "p95": 0.23065781593322754 }, { "label": "current_group_partial", "mean": 0.11520501971244812, "median": 0.10386130958795547, "p05": 0.03912060335278511, "p95": 0.22735844552516937 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 1.9781701564788818, "group": 8, "latest_source_mass": 0.16112206876277924, "layer": 32, "max_source_mass": 0.22251944243907928, "mixer_index": 63, "normalized_entropy": 0.9003040366848136, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.09446848928928375, "median": 0.074150450527668, "p05": 0.042592622339725494, "p95": 0.19443637132644653 }, { "label": "completed_group_1", "mean": 0.07056106626987457, "median": 0.06868506222963333, "p05": 0.04818308353424072, "p95": 0.0982733890414238 }, { "label": "completed_group_2", "mean": 0.08033448457717896, "median": 0.08047192543745041, "p05": 0.048296794295310974, "p95": 0.11253741383552551 }, { "label": "completed_group_3", "mean": 0.0658978745341301, "median": 0.06256154924631119, "p05": 0.04223886504769325, "p95": 0.0967884361743927 }, { "label": "completed_group_4", "mean": 0.058234501630067825, "median": 0.0563921295106411, "p05": 0.03529864549636841, "p95": 0.08846980333328247 }, { "label": "completed_group_5", "mean": 0.04316519945859909, "median": 0.042268507182598114, "p05": 0.029443470761179924, "p95": 0.05996086448431015 }, { "label": "completed_group_6", "mean": 0.2036968618631363, "median": 0.19645483791828156, "p05": 0.09398645162582397, "p95": 0.3395216464996338 }, { "label": "completed_group_7", "mean": 0.22251944243907928, "median": 0.2130982130765915, "p05": 0.11867167800664902, "p95": 0.35465875267982483 }, { "label": "current_group_partial", "mean": 0.16112206876277924, "median": 0.14023703336715698, "p05": 0.06008770316839218, "p95": 0.3247341811656952 } ], "sources": 9 }, { "branch": "output", "entropy_mean": 1.6036605834960938, "group": null, "latest_source_mass": 0.31639498472213745, "layer": null, "max_source_mass": 0.3412485122680664, "mixer_index": null, "normalized_entropy": 0.7298573846467136, "offset": null, "source_summaries": [ { "label": "embedding", "mean": 0.09759747982025146, "median": 0.09915056824684143, "p05": 0.025461740791797638, "p95": 0.16398483514785767 }, { "label": "completed_group_1", "mean": 0.04215940088033676, "median": 0.04050712287425995, "p05": 0.025720715522766113, "p95": 0.06398092210292816 }, { "label": "completed_group_2", "mean": 0.041145481169223785, "median": 0.03881259262561798, "p05": 0.023510465398430824, "p95": 0.0662468671798706 }, { "label": "completed_group_3", "mean": 0.029554028064012527, "median": 0.027420252561569214, "p05": 0.01627083122730255, "p95": 0.04924973472952843 }, { "label": "completed_group_4", "mean": 0.0289996936917305, "median": 0.027226522564888, "p05": 0.01560397818684578, "p95": 0.048351090401411057 }, { "label": "completed_group_5", "mean": 0.03647579252719879, "median": 0.0347352996468544, "p05": 0.020744025707244873, "p95": 0.058257561177015305 }, { "label": "completed_group_6", "mean": 0.06642461568117142, "median": 0.057809896767139435, "p05": 0.022837381809949875, "p95": 0.14078694581985474 }, { "label": "completed_group_7", "mean": 0.3412485122680664, "median": 0.3336990475654602, "p05": 0.0819513276219368, "p95": 0.6423859596252441 }, { "label": "completed_group_8", "mean": 0.31639498472213745, "median": 0.30713334679603577, "p05": 0.05841828137636185, "p95": 0.6156357526779175 } ], "sources": 9 } ], "mode": "uniform_depth_all", "optimizer_state_after": "ed03e6fbd4a12d8b063dcb22e0437754285f54d585374cd52fbd534f05d24637", "optimizer_state_before": "ed03e6fbd4a12d8b063dcb22e0437754285f54d585374cd52fbd534f05d24637", "optimizer_state_unchanged": true, "positions": { "attention_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "5e6ecbcd78543fc42d4098a51169dca446d404ce384015643e84d2c78433c949", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.0933690071105957, 0.24537226557731628, 0.07698935270309448, 0.0523640513420105, 0.09380833059549332, 0.03647654876112938, 0.039120711386203766, 0.06371714919805527, 0.05156813934445381, 0.03728010877966881, 0.03983496502041817, 0.040284499526023865, 0.046871740370988846, 0.04314620420336723, 0.04033593460917473, 0.04471700266003609, 0.08397205173969269, 0.0528092160820961, 0.04443390294909477, 0.07560808956623077, 0.03708760067820549, 0.03599191829562187, 0.033304885029792786, 0.04061548039317131, 0.025487789884209633, 0.021565141156315804, 0.02342291921377182, 0.017244853079319, 0.024166326969861984, 0.016819356009364128, 0.021590016782283783, 0.026201345026493073 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.9843463405777584e-05, "first_to_last_ratio": 1.726461496638441, "imbalance_abs_log_ratio": 0.5460739362024395, "last_quartile_mean": 1.1493719057398266e-05, "mean": 1.365886706139463e-05, "normalized": [ 1.7175809809832567, 1.841085956997995, 1.5856650780422397, 1.4117328942326883, 1.3850619805193556, 1.3059240514218338, 1.240189963407468, 1.1350778806219197, 1.0683250207815616, 0.9892315713951353, 0.9427237719154575, 0.9054921895502139, 0.8890536069806895, 0.856278718542603, 0.8363872319332963, 0.8150102694634626, 0.8192882455095108, 0.8012437322929742, 0.8014534794336197, 0.7905947034964168, 0.8013195741955759, 0.787926653281012, 0.7760431793287634, 0.7654366992378798, 1.0622548719448865, 1.040165634819978, 1.0053520041748172, 0.9472312712611266, 0.7283781749102672, 0.6871697867583846, 0.6410501859548857, 0.6202706366107255 ], "peak_layer": 2, "peak_normalized": 1.841085956997995, "population_cv": 0.30422377034429426, "rest_mean": 1.4067125044271557e-05, "spike_contrast": 0.8142583447442704, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.1454273953859229e-05, "top_five_layers": [ 2, 1, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 2.3460210286430083e-05, 2.5147148335236125e-05, 2.1658388504874893e-05, 1.9282671928522177e-05, 1.8918377463705838e-05, 1.7837443010648713e-05, 1.6939589841058478e-05, 1.5503877875744365e-05, 1.4592109437217005e-05, 1.3511782526620664e-05, 1.2876538676209748e-05, 1.236799744219752e-05, 1.2143465028202627e-05, 1.1695797184074763e-05, 1.1424102012824733e-05, 1.113211692427285e-05, 1.1190549230377655e-05, 1.0944081623165403e-05, 1.0946946531475987e-05, 1.0798627954500262e-05, 1.0945117537630722e-05, 1.0762185411294922e-05, 1.0599870620353613e-05, 1.0454998118802905e-05, 1.4509198081213981e-05, 1.4207484127837233e-05, 1.3731969374930486e-05, 1.2938106010551564e-05, 9.948820661520585e-06, 9.385960765939672e-06, 8.756019269640092e-06, 8.472194167552516e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 7.914939033071278e-05, "first_to_last_ratio": 1.7192761584455627, "imbalance_abs_log_ratio": 0.5419033641351061, "last_quartile_mean": 4.6036461299081566e-05, "mean": 5.459262786189356e-05, "normalized": [ 1.7241794077033439, 1.8409283164897874, 1.5899715734737732, 1.4105364899762889, 1.3785034260791753, 1.2918378676826083, 1.2297131902035034, 1.1328760421453867, 1.0698767990750255, 0.9917311914967222, 0.9453692193632256, 0.9099605748485345, 0.8959140145556254, 0.8626522006632991, 0.8400445735074902, 0.8167598977074292, 0.82163777996241, 0.8002639673389168, 0.7991874200113684, 0.7879667381444585, 0.7970945666923349, 0.7843979054532264, 0.7719455414906756, 0.7604713662140478, 1.0567226635305242, 1.0350451119581627, 1.0015649898612529, 0.9453760165051265, 0.7341808876435505, 0.6945539501928557, 0.6495600028248955, 0.6291763072049747 ], "peak_layer": 2, "peak_normalized": 1.8409283164897874, "population_cv": 0.3029903783236571, "rest_mean": 5.6269567721756175e-05, "spike_contrast": 0.8092678593837654, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.553715261863544e-05, "top_five_layers": [ 2, 1, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 9.41274847718887e-05, 0.00010050111450254917, 8.680072642164305e-05, 7.700489368289709e-05, 7.525612454628572e-05, 7.052482396829873e-05, 6.71332745696418e-05, 6.184668018249795e-05, 5.840738594997674e-05, 5.4141211876412854e-05, 5.161018998478539e-05, 4.967713903170079e-05, 4.891030039289035e-05, 4.709445056505501e-05, 4.58602407888975e-05, 4.4589069148059934e-05, 4.485536555876024e-05, 4.368851296021603e-05, 4.3629741412587464e-05, 4.3017174903070554e-05, 4.351548705017194e-05, 4.282234294805676e-05, 4.214253567624837e-05, 4.1516130295349285e-05, 5.7689267123350874e-05, 5.650583261740394e-05, 5.467806477099657e-05, 5.1610561058623716e-05, 4.008086398243904e-05, 3.7917525332886726e-05, 3.546118750819005e-05, 3.434838799876161e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.07019046926870942, "first_to_last_ratio": 1.7192761309643136, "imbalance_abs_log_ratio": 0.5419033481509088, "last_quartile_mean": 0.040825593983754516, "mean": 0.048413287790026516, "normalized": [ 1.7241793415740942, 1.8409282128727242, 1.5899715027984953, 1.410536586356086, 1.3785034200197028, 1.2918378607824799, 1.2297132259303432, 1.1328760294889055, 1.0698768072772828, 0.9917312032390512, 0.9453692242028005, 0.9099605938774525, 0.8959140252770564, 0.8626522352939275, 0.8400445444843937, 0.8167598679640768, 0.8216378124018651, 0.8002639776814882, 0.799187479603078, 0.7879667376954292, 0.7970945795863034, 0.7843979041802996, 0.7719455376662867, 0.7604713300924276, 1.0567226778987167, 1.035045130675446, 1.0015649631693344, 0.9453760725465444, 0.7341808462499567, 0.6945539439690372, 0.6495600177802529, 0.6291763073646611 ], "peak_layer": 2, "peak_normalized": 1.8409282128727242, "population_cv": 0.30299036400954366, "rest_mean": 0.049900414836075574, "spike_contrast": 0.8092678562697402, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.04038280174136162, "top_five_layers": [ 2, 1, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.08347319066524506, 0.0891253873705864, 0.0769757479429245, 0.06828871369361877, 0.06673788279294968, 0.06254211813211441, 0.0595344603061676, 0.054846253246068954, 0.05179625377058983, 0.04801296815276146, 0.045768432319164276, 0.04405418410897255, 0.04337414354085922, 0.04176383092999458, 0.04066931828856468, 0.03954203054308891, 0.03977818787097931, 0.038743410259485245, 0.03869129344820976, 0.03814806044101715, 0.0385899692773819, 0.03797528147697449, 0.037372421473264694, 0.03681691735982895, 0.05115941911935806, 0.05010993778705597, 0.048489052802324295, 0.04576876387000084, 0.03554410859942436, 0.03362563997507095, 0.031447336077690125, 0.03046049363911152 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 3.1499947965585307e-06, "first_to_last_ratio": 1.0342345204222718, "imbalance_abs_log_ratio": 0.033661559288008014, "last_quartile_mean": 3.045725833317192e-06, "mean": 2.7674113027842395e-06, "normalized": [ 2.1814481803150345, 1.2474055539433142, 1.1222180172833975, 1.0031956884625697, 0.9117870624730428, 0.8816544633234661, 0.8770498242321535, 0.8812090677679382, 0.8630917141524694, 0.8317024603226125, 0.8236332501585274, 0.8250898850605665, 0.8170645489467768, 0.8225184877719931, 0.8353122922959889, 0.8506602402685544, 0.8463384819921621, 0.8686007009434122, 0.8872533344047147, 0.9073899624286337, 0.931692735524339, 0.9651981317500963, 0.9968816046325903, 1.0170559446210892, 1.3744139458516371, 1.3637523051464024, 1.333599083549964, 1.2602900643690447, 0.9346898918552929, 0.8957006501483127, 0.8422479251718212, 0.7998545008320819 ], "peak_layer": 1, "peak_normalized": 2.1814481803150345, "population_cv": 0.27076976140585923, "rest_mean": 2.738174897684884e-06, "spike_contrast": 1.0683349309768633, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.92528789032076e-06, "top_five_layers": [ 1, 25, 26, 27, 28 ], "top_five_spike_overlap": 1 }, "values": [ 6.036964350641938e-06, 3.4520842291385634e-06, 3.1056388252181932e-06, 2.7762550871557323e-06, 2.523289822420338e-06, 2.439900526951533e-06, 2.4271575966849923e-06, 2.438667934256955e-06, 2.388529765084968e-06, 2.3016627892502584e-06, 2.2793319658376276e-06, 2.2833630737295607e-06, 2.2611536678596167e-06, 2.276246959809214e-06, 2.311652679054532e-06, 2.3541267637483543e-06, 2.342166681046365e-06, 2.403775397397112e-06, 2.455394906064612e-06, 2.5111212380579673e-06, 2.5783770070120227e-06, 2.671100219231448e-06, 2.75878142019792e-06, 2.814612116708304e-06, 3.8035686884541065e-06, 3.7740635434602154e-06, 3.690617177198874e-06, 3.487740968921571e-06, 2.5866713713185163e-06, 2.4787721031316323e-06, 2.3308464278670726e-06, 2.213526386185549e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 5.945006205365644e-05, "first_to_last_ratio": 2.017462962034897, "imbalance_abs_log_ratio": 0.7018407626464352, "last_quartile_mean": 2.9467734066201956e-05, "mean": 3.904375290630924e-05, "normalized": [ 1.7888984161537487, 1.9066934939679154, 1.6645224349155732, 1.4966721268131649, 1.4710765048265346, 1.374893360589443, 1.2914915816094943, 1.186970868429721, 1.113397210469164, 1.0295988702815844, 0.97642332943856, 0.9358540743157573, 0.9225709510456688, 0.8813984681836262, 0.8530918620954848, 0.8242460408502565, 0.8352221953845033, 0.8028126343581958, 0.8023765661129623, 0.7855184286477619, 0.79058036101754, 0.7659725081479444, 0.7434004324270638, 0.7184276053445235, 0.9558735202049503, 0.9292908745170894, 0.8990855080226723, 0.8518440360398932, 0.6531104523579828, 0.6161491960796108, 0.5764091694500679, 0.5561269179015412 ], "peak_layer": 2, "peak_normalized": 1.9066934939679154, "population_cv": 0.3438795276542928, "rest_mean": 4.052704759858359e-05, "spike_contrast": 0.7657592498574302, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.1033961568027736e-05, "top_five_layers": [ 2, 1, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 6.984530773479491e-05, 7.444446964655071e-05, 6.498920265585184e-05, 5.843569670105353e-05, 5.7436347560724244e-05, 5.368099664337933e-05, 5.042467819293961e-05, 4.634379729395732e-05, 4.347120557213202e-05, 4.019940388388932e-05, 3.812323120655492e-05, 3.653925523394719e-05, 3.602063225116581e-05, 3.4413104003760964e-05, 3.3307907870039344e-05, 3.2181658752961084e-05, 3.261020901845768e-05, 3.1344818125944585e-05, 3.1327792385127395e-05, 3.066958743147552e-05, 3.0867224268149585e-05, 2.9906441341154277e-05, 2.9025142794125713e-05, 2.8050109904143028e-05, 3.732088953256607e-05, 3.628300328273326e-05, 3.510367241688073e-05, 3.325918805785477e-05, 2.549988312239293e-05, 2.4056776965153404e-05, 2.2505177184939384e-05, 2.17132819670951e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 4.342395959611167e-05, "first_to_last_ratio": 2.3866820108839386, "imbalance_abs_log_ratio": 0.8699041214250928, "last_quartile_mean": 1.8194279505223676e-05, "mean": 2.7162555994664217e-05, "normalized": [ 1.8780523787026802, 1.9911893018422189, 1.7524284245860786, 1.5914635463665663, 1.574190130461127, 1.4630804262340853, 1.3267938845882294, 1.2121607892452353, 1.1372748993874344, 1.0531060719751173, 0.9945687865408062, 0.9423957577320254, 0.9298315140367374, 0.8843077525462589, 0.8579826354401896, 0.8310938587943976, 0.8495195054281806, 0.8070134017943003, 0.814017660109403, 0.7897681119440687, 0.7909520849347184, 0.7575702075438251, 0.725297836064079, 0.6873055624902523, 0.8503252499430842, 0.8316831666177978, 0.8092299358450645, 0.7657418980504435, 0.5785094272044417, 0.5451757324247911, 0.499011734860947, 0.47895832626541507 ], "peak_layer": 2, "peak_normalized": 1.9911893018422189, "population_cv": 0.39213983031657196, "rest_mean": 2.8358260823114068e-05, "spike_contrast": 0.7301487933335571, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.0705749921035022e-05, "top_five_layers": [ 2, 1, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 5.101270289742388e-05, 5.408579090726562e-05, 4.7600435209460557e-05, 4.322821769164875e-05, 4.275902756489813e-05, 3.974100400228053e-05, 3.603911318350583e-05, 3.292538531241007e-05, 3.08912931359373e-05, 2.860505264834501e-05, 2.701503035495989e-05, 2.5597877538530156e-05, 2.5256600565626286e-05, 2.4020058845053427e-05, 2.3305001377593726e-05, 2.257463347632438e-05, 2.3075121134752408e-05, 2.1920546714682132e-05, 2.2110800273367204e-05, 2.1452120563481003e-05, 2.1484280296135694e-05, 2.057754318229854e-05, 1.9700943084899336e-05, 1.8668975826585665e-05, 2.309700721525587e-05, 2.2590640583075583e-05, 2.1980753444950096e-05, 2.0799507183255628e-05, 1.5713794709881768e-05, 1.4808366358920466e-05, 1.3554434190155007e-05, 1.300973235629499e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00016796771706140134, "first_to_last_ratio": 1.6975867729858307, "imbalance_abs_log_ratio": 0.5292076972507559, "last_quartile_mean": 9.894499635265674e-05, "mean": 0.00011761535029108927, "normalized": [ 1.6704274564617343, 1.7775222554939007, 1.5539435667351813, 1.4136060640114756, 1.3694591378553722, 1.2875366088844244, 1.2254186807996688, 1.126970253531922, 1.0609391629755696, 0.9813912807517243, 0.9455189338588661, 0.9218292559911828, 0.9168400604471353, 0.8780476886557013, 0.8518972517012092, 0.8289796773621715, 0.8347336766819297, 0.8190795421000308, 0.8186461965974139, 0.80096273069962, 0.811407996663982, 0.8000351053160739, 0.7905925037209359, 0.7841412542712713, 1.0905788818329314, 1.0687938208581202, 1.0381744504119732, 0.9857028365190887, 0.6941958019263929, 0.6517718030498588, 0.6078197431797288, 0.5930363206534093 ], "peak_layer": 2, "peak_normalized": 1.7775222554939007, "population_cv": 0.29323151068952025, "rest_mean": 0.00012076589202344487, "spike_contrast": 0.8330367395194549, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00010060242493636906, "top_five_layers": [ 2, 1, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00019646791042760015, 0.0002090639027301222, 0.000182767616934143, 0.00016626177239231765, 0.0001610694162081927, 0.0001514340692665428, 0.00014412804739549756, 0.00013254900113679469, 0.00012478273129090667, 0.0001154266792582348, 0.0001112075406126678, 0.00010842127085197717, 0.00010783446487039328, 0.00010327188647352159, 0.00010019619367085397, 9.750073513714597e-05, 9.817749378271401e-05, 9.633632726036012e-05, 9.628535917727277e-05, 9.42055121413432e-05, 9.543403575662524e-05, 9.409640915691853e-05, 9.298581426264718e-05, 9.222704829880968e-05, 0.0001282688172068447, 0.00012570655962917954, 0.0001221052516484633, 0.00011593378440011293, 8.164808241417632e-05, 7.665836892556399e-05, 7.148893200792372e-05, 6.975017458898947e-05 ] } } }, "mlp_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "c0b8018c349b1bd09e5d17858eb312a2323da0e92de9651e2bb08ae4adf115c7", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.13583384454250336, 0.0415157787501812, 0.018140379339456558, 0.039518751204013824, 0.022877270355820656, 0.015532293356955051, 0.02314678393304348, 0.05432315170764923, 0.03743875026702881, 0.04006272554397583, 0.045583780854940414, 0.05995568633079529, 0.038651175796985626, 0.04559725895524025, 0.05632574483752251, 0.07952121645212173, 0.042337141931056976, 0.05044656619429588, 0.04418695345520973, 0.06310147047042847, 0.029010163620114326, 0.04586305841803551, 0.06033914536237717, 0.07656749337911606, 0.045884523540735245, 0.051604680716991425, 0.06502768397331238, 0.06792232394218445, 0.060306549072265625, 0.08025261014699936, 0.09663844108581543, 0.12230555713176727 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.9286482711322606e-05, "first_to_last_ratio": 1.7521178183449044, "imbalance_abs_log_ratio": 0.5608252381669183, "last_quartile_mean": 1.1007526154571678e-05, "mean": 1.3294077234604629e-05, "normalized": [ 1.5831089740245814, 1.8642713266768112, 1.6087569313591699, 1.407175518493288, 1.414036437386145, 1.327863170210972, 1.2500486676722316, 1.1507992495720307, 1.0468180783427234, 0.9815178464379728, 0.9385355434779077, 0.9054461784802859, 0.897623365408445, 0.8647601755285843, 0.8426881932107443, 0.8244111776071582, 0.8360622063670075, 0.8172788622556931, 0.8192465720975091, 0.8139036125249431, 0.8167375060225283, 0.799846890528267, 0.7893987768003575, 0.7756465634089913, 1.0687136905899075, 1.0326901378652278, 0.974197393889338, 0.9219230973813526, 0.7056634256328721, 0.6594305223868727, 0.6385505089367735, 0.6228493994233082 ], "peak_layer": 2, "peak_normalized": 1.8642713266768112, "population_cv": 0.2995364157375371, "rest_mean": 1.3663188063354162e-05, "spike_contrast": 0.8271040921750225, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.1300878759357147e-05, "top_five_layers": [ 2, 3, 1, 5, 4 ], "top_five_spike_overlap": 0 }, "values": [ 2.1045972971478477e-05, 2.4783767003100365e-05, 2.138693889719434e-05, 1.8707100025494583e-05, 1.8798309611156583e-05, 1.7652715541771613e-05, 1.661824353504926e-05, 1.5298814105335623e-05, 1.3916480384068564e-05, 1.3048374057689216e-05, 1.2476964002416935e-05, 1.2037071428494528e-05, 1.19330743473256e-05, 1.1496188562887255e-05, 1.1202761925233062e-05, 1.0959785868180916e-05, 1.111467554437695e-05, 1.0864968317036983e-05, 1.0891127203649376e-05, 1.0820097486430313e-05, 1.0857771485461853e-05, 1.0633226338541135e-05, 1.0494328307686374e-05, 1.0311505320714787e-05, 1.4207562344381586e-05, 1.372866245219484e-05, 1.2951055396115407e-05, 1.2256116860953625e-05, 9.381144081999082e-06, 8.766520295466762e-06, 8.488939784001559e-06, 8.280208021460567e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 7.694901069044136e-05, "first_to_last_ratio": 1.7432195508580701, "imbalance_abs_log_ratio": 0.5557337200845022, "last_quartile_mean": 4.414189288581838e-05, "mean": 5.3159109029365936e-05, "normalized": [ 1.5913307482959713, 1.8632629984963744, 1.6128840124944015, 1.4043056655908324, 1.4066439752831947, 1.312671956897304, 1.2393367123270258, 1.1497445762962974, 1.0498174919276073, 0.9853903121467731, 0.942472392433832, 0.9105602958514721, 0.9050181706519704, 0.8709205812100463, 0.846131336743431, 0.8259289925089213, 0.8378922989837381, 0.8154040667738044, 0.8159779682336543, 0.8104003933613405, 0.811492626523659, 0.7953174472433975, 0.7839918927008893, 0.770118478469589, 1.0630407673718132, 1.0283068631464782, 0.9709725606615203, 0.9201945998839314, 0.7128592499056251, 0.6675478613569114, 0.6476460181133628, 0.6324166881148308 ], "peak_layer": 2, "peak_normalized": 1.8632629984963744, "population_cv": 0.29800433084248273, "rest_mean": 5.468701756736091e-05, "spike_contrast": 0.8211894691254099, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.490840292419307e-05, "top_five_layers": [ 2, 3, 1, 5, 4 ], "top_five_spike_overlap": 0 }, "values": [ 8.459372475044802e-05, 9.904940088745207e-05, 8.57394770719111e-05, 7.465163798769936e-05, 7.477594044758007e-05, 6.978047167649493e-05, 6.588203541468829e-05, 6.111939728725702e-05, 5.580736251431517e-05, 5.238247103989124e-05, 5.010099266655743e-05, 4.840457404498011e-05, 4.81099596072454e-05, 4.6297362132463604e-05, 4.4979587983107194e-05, 4.3905649363296106e-05, 4.4541608076542616e-05, 4.334615368861705e-05, 4.337666177889332e-05, 4.308016286813654e-05, 4.3138225009897724e-05, 4.2278366890968755e-05, 4.167631050222553e-05, 4.093881216249429e-05, 5.651030005537905e-05, 5.466387665364891e-05, 5.161603621672839e-05, 4.891672506346367e-05, 3.789496258832514e-05, 3.5486249544192106e-05, 3.442828528932296e-05, 3.36187076754868e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.06823915196582675, "first_to_last_ratio": 1.7432195265093744, "imbalance_abs_log_ratio": 0.5557337061168434, "last_quartile_mean": 0.03914547245949507, "mean": 0.04714202950708568, "normalized": [ 1.5913307755497357, 1.8632630471719813, 1.6128838999775155, 1.4043057047349041, 1.4066439863295674, 1.3126719339409767, 1.2393367378121998, 1.1497445183869146, 1.0498174736221622, 0.9853902690216583, 0.9424724102749755, 0.9105602795625451, 0.9050181017538224, 0.8709205983647982, 0.8461313364627144, 0.8259290260119257, 0.8378923510845062, 0.8154040720642544, 0.8159779348875205, 0.8104004339333831, 0.8114926066241517, 0.7953174508414004, 0.7839919179536974, 0.770118456257048, 1.0630408155932525, 1.0283068634051942, 0.9709725780129965, 0.9201945686959736, 0.7128592424266932, 0.6675478652203702, 0.6476460385736946, 0.6324167054474666 ], "peak_layer": 2, "peak_normalized": 1.8632630471719813, "population_cv": 0.29800432900072793, "rest_mean": 0.048496994017450896, "spike_contrast": 0.8211894769556856, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.03982522115111351, "top_five_layers": [ 2, 3, 1, 5, 4 ], "top_five_spike_overlap": 0 }, "values": [ 0.07501856237649918, 0.08783800154924393, 0.07603462040424347, 0.0662018209695816, 0.06631205230951309, 0.06188201904296875, 0.058424849063158035, 0.054201290011405945, 0.04949052631855011, 0.04645329713821411, 0.04443006217479706, 0.04292565956711769, 0.04266439005732536, 0.04105696454644203, 0.039888348430395126, 0.03893597051501274, 0.03949994593858719, 0.0384398028254509, 0.03846685588359833, 0.03820392116904259, 0.03825540840625763, 0.03749287873506546, 0.0369589701294899, 0.03630494698882103, 0.05011390149593353, 0.04847647249698639, 0.045773617923259735, 0.04337983950972557, 0.033605631440877914, 0.03146956115961075, 0.030531348660588264, 0.029813406988978386 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 2.992666935597299e-06, "first_to_last_ratio": 1.0269747300979142, "imbalance_abs_log_ratio": 0.02661732509152257, "last_quartile_mean": 2.9140609285605024e-06, "mean": 2.6725837187768775e-06, "normalized": [ 2.020771646774043, 1.2377394447364218, 1.1093031629899148, 0.9724379415861507, 0.9355746131312372, 0.9012237595282857, 0.8951394390788443, 0.8859336672438388, 0.8578676590539637, 0.8345125837496935, 0.8291747232200378, 0.8280299358162725, 0.8261494081131788, 0.8339287047401779, 0.8457088019356572, 0.8617084310027588, 0.8674420667258089, 0.8883581731469679, 0.9102343662098846, 0.9354629929563161, 0.9548306246795014, 0.9876911107334666, 1.021571151799822, 1.036377840247453, 1.398185278855825, 1.3626778973124167, 1.3104461212520941, 1.2283253209133203, 0.9133467144256953, 0.864309555414376, 0.8394556900021886, 0.8060811726243864 ], "peak_layer": 1, "peak_normalized": 2.020771646774043, "population_cv": 0.24560684573195382, "rest_mean": 2.6331229205593615e-06, "spike_contrast": 1.0959123885255055, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.885672029151465e-06, "top_five_layers": [ 1, 25, 26, 27, 2 ], "top_five_spike_overlap": 1 }, "values": [ 5.4006814025342464e-06, 3.3079622880904935e-06, 2.9647055725945393e-06, 2.5989218102040468e-06, 2.5004014787555207e-06, 2.408595946690184e-06, 2.392335090917186e-06, 2.3677318949921755e-06, 2.292723138452857e-06, 2.230304744443856e-06, 2.216038865299197e-06, 2.2129793251224328e-06, 2.2079534574004356e-06, 2.2287442789092893e-06, 2.2602275748795364e-06, 2.3029879230307415e-06, 2.3183115445135627e-06, 2.3742115899949567e-06, 2.4326775474037277e-06, 2.500103164493339e-06, 2.551864781707991e-06, 2.639687181726913e-06, 2.7302344278723467e-06, 2.769806542346487e-06, 3.7367672121035866e-06, 3.6418707622942748e-06, 3.502276967992657e-06, 3.2828022540343227e-06, 2.4409955585724674e-06, 2.309939645783743e-06, 2.243515609734459e-06, 2.1543194179685088e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 5.819382158733788e-05, "first_to_last_ratio": 2.050273018319789, "imbalance_abs_log_ratio": 0.7179729639508818, "last_quartile_mean": 2.83834499441582e-05, "mean": 3.819049175035616e-05, "normalized": [ 1.6894578900891235, 1.9241457487811389, 1.6897238525411336, 1.495305300121677, 1.496280749802402, 1.3930872231636864, 1.2995495052590544, 1.20267172952676, 1.0917712930814951, 1.0214370844571112, 0.9712476263973425, 0.9342054963243545, 0.9289466413366485, 0.8873198977642526, 0.8563143151783298, 0.8310063494189237, 0.8486873274696448, 0.815916028547597, 0.816985403363595, 0.8077040568067506, 0.8024742557401989, 0.7739841711737009, 0.7515294433597944, 0.724590695294805, 0.9569948205253259, 0.9222432825444, 0.8748347134035187, 0.8323495931638362, 0.6311765279991477, 0.5936525791287859, 0.574350201760133, 0.5600561964753319 ], "peak_layer": 2, "peak_normalized": 1.9241457487811389, "population_cv": 0.34212280954756363, "rest_mean": 3.9591413210641973e-05, "spike_contrast": 0.7735393455614457, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.0625515864812766e-05, "top_five_layers": [ 2, 3, 1, 5, 4 ], "top_five_spike_overlap": 0 }, "values": [ 6.452122761402279e-05, 7.348407234530896e-05, 6.453138485085219e-05, 5.7106444728560746e-05, 5.7143697631545365e-05, 5.3202686103759333e-05, 4.9630434659775347e-05, 4.59306247648783e-05, 4.1695282561704516e-05, 3.900918454746716e-05, 3.709242446348071e-05, 3.567776730051264e-05, 3.5476929042488337e-05, 3.388718323549256e-05, 3.270306478952989e-05, 3.173654113197699e-05, 3.2411786378361285e-05, 3.1160234357230365e-05, 3.120107430731878e-05, 3.084661511820741e-05, 3.064688644371927e-05, 2.955883610411547e-05, 2.8701279006781988e-05, 2.7672474971041083e-05, 3.6548102798406035e-05, 3.522092447383329e-05, 3.3410367905162275e-05, 3.178784027113579e-05, 2.4104941985569894e-05, 2.2671883925795555e-05, 2.1934716642135754e-05, 2.138882155122701e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 4.280550319890608e-05, "first_to_last_ratio": 2.4511850562189483, "imbalance_abs_log_ratio": 0.89657160402913, "last_quartile_mean": 1.746318707773753e-05, "mean": 2.6669155204217532e-05, "normalized": [ 1.8016469213187563, 1.9979965076104602, 1.7802215893582338, 1.6026062094418312, 1.601795652455447, 1.482400144562061, 1.3412434596559013, 1.2325422021936074, 1.1215677839064897, 1.044684639475341, 0.9879507658587825, 0.9385570573180131, 0.9374768148449106, 0.8904771959018412, 0.8607364932284278, 0.8355215140718881, 0.8610488755069231, 0.8184961342479893, 0.8302957950693817, 0.8152532242450765, 0.8016879896985464, 0.7642274206077754, 0.7257649774364763, 0.6873333632586007, 0.8503651969651687, 0.8258800192016202, 0.7815531102946032, 0.7478452208900362, 0.5565130873808856, 0.5144350239532156, 0.49036538964637755, 0.4715102203953315 ], "peak_layer": 2, "peak_normalized": 1.9979965076104602, "population_cv": 0.39439468267452726, "rest_mean": 2.782543203836583e-05, "spike_contrast": 0.7340500687160684, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.0425260299816727e-05, "top_five_layers": [ 2, 1, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 4.80484013678506e-05, 5.3284878958947957e-05, 4.747700586449355e-05, 4.274015373084694e-05, 4.27185368607752e-05, 3.953435953008011e-05, 3.5769829992204905e-05, 3.287085928604938e-05, 2.991126530105248e-05, 2.7860856789629906e-05, 2.6347812308813445e-05, 2.5030523829627782e-05, 2.5001714675454423e-05, 2.3748274543322623e-05, 2.2955115127842873e-05, 2.2282652935246006e-05, 2.2963446099311113e-05, 2.1828600438311696e-05, 2.2143287424114533e-05, 2.1742114768130705e-05, 2.138034142262768e-05, 2.0381299691507593e-05, 1.935553882503882e-05, 1.833060014178045e-05, 2.2678521418129094e-05, 2.2025522412150167e-05, 2.0843361198785715e-05, 1.994440026464872e-05, 1.4841733900539111e-05, 1.3719547496293671e-05, 1.3077630683255848e-05, 1.2574779248097911e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.0001629307516850531, "first_to_last_ratio": 1.7143242165586177, "imbalance_abs_log_ratio": 0.5390189601396663, "last_quartile_mean": 9.504080389888259e-05, "mean": 0.00011444387678238854, "normalized": [ 1.5559615617438098, 1.785869456292719, 1.5834930459757501, 1.3910723566520669, 1.402568158270034, 1.3078266248149302, 1.239528906365914, 1.123070481387428, 1.0318816108000253, 0.979452304814572, 0.9467605586943024, 0.9155518013723865, 0.9213121632035975, 0.8879781658254625, 0.8586858628608119, 0.8366871996511438, 0.8544112236236473, 0.8355171986712887, 0.8376016860986925, 0.8224986728524569, 0.8250120483204031, 0.8118832806008449, 0.8030014962783038, 0.798712170836371, 1.098625388915964, 1.067521151596813, 1.0137573718534312, 0.96062548037494, 0.6714196499508523, 0.6232967079479287, 0.6123656576587695, 0.5960505556943392 ], "peak_layer": 2, "peak_normalized": 1.785869456292719, "population_cv": 0.28815180066158774, "rest_mean": 0.00011725311627809856, "spike_contrast": 0.8466639237979695, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.92739835055545e-05, "top_five_layers": [ 2, 3, 1, 5, 4 ], "top_five_spike_overlap": 0 }, "values": [ 0.00017807027325034142, 0.00020438182400539517, 0.00018122108303941786, 0.000159199713380076, 0.0001605153374839574, 0.00014967274910304695, 0.0001418564934283495, 0.00012852853978984058, 0.00011809253192041069, 0.00011209231888642535, 0.00010835094872163609, 0.00010477929754415527, 0.00010543853568378836, 0.00010162366379518062, 9.827133908402175e-05, 9.575372678227723e-05, 9.778213279787451e-05, 9.561982733430341e-05, 9.585838415659964e-05, 9.412993676960468e-05, 9.44175772019662e-05, 9.291507012676448e-05, 9.189860429614782e-05, 9.140771726379171e-05, 0.00012573094863910228, 0.00012217125913593918, 0.00011601832375163212, 0.00010993770411005244, 7.68398676882498e-05, 7.133249164326116e-05, 7.008149987086654e-05, 6.821433635195717e-05 ] } } }, "post_attention_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "90a633f12d984935028fe5d7212be48e7da7b4a0806751939a551938d16dc540", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.0933690071105957, 0.35791492462158203, 0.3940892517566681, 0.4057151973247528, 0.09380833059549332, 0.11286329478025436, 0.13231655955314636, 0.1621972620487213, 0.05156813934445381, 0.08533531427383423, 0.12003090232610703, 0.16063299775123596, 0.046871740370988846, 0.079384446144104, 0.11459959298372269, 0.15625107288360596, 0.08397205173969269, 0.11654024571180344, 0.15294702351093292, 0.21572710573673248, 0.03708760067820549, 0.06553087383508682, 0.09390194714069366, 0.12852425873279572, 0.025487789884209633, 0.06059965491294861, 0.08860157430171967, 0.12100546807050705, 0.024166326969861984, 0.06877554953098297, 0.12682415544986725, 0.2025238573551178 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.9843389054585714e-05, "first_to_last_ratio": 1.726430471786405, "imbalance_abs_log_ratio": 0.5460559658446521, "last_quartile_mean": 1.149388253907091e-05, "mean": 1.3658895937851412e-05, "normalized": [ 1.7175876041050318, 1.841018674620317, 1.585694486202753, 1.411695151644958, 1.3850332168748785, 1.3059402010459964, 1.2401880073676108, 1.135093326054235, 1.068313440145743, 0.9892263504933855, 0.9427245755130007, 0.9054902086506667, 0.8890448690363575, 0.8562741116495445, 0.8363941199286294, 0.8149952291886035, 0.8193029602456535, 0.801251027516921, 0.801446391583471, 0.7906043517524775, 0.8013096900058885, 0.7879301812423499, 0.7760469987581269, 0.7654407408493056, 1.0622634797767363, 1.0401912022592665, 1.0053793764566537, 0.9472407881284585, 0.7283795648312444, 0.6871696657284989, 0.6410522266009181, 0.6202777817423186 ], "peak_layer": 2, "peak_normalized": 1.841018674620317, "population_cv": 0.3042176211627124, "rest_mean": 1.4067149667998482e-05, "spike_contrast": 0.814260604699103, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.145432579505723e-05, "top_five_layers": [ 2, 1, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 2.3460350348614156e-05, 2.5146282496280037e-05, 2.1658835976268165e-05, 1.928219717228785e-05, 1.891802457976155e-05, 1.7837701307144016e-05, 1.6939598936005495e-05, 1.550412162032444e-05, 1.4591982107958756e-05, 1.351173978036968e-05, 1.2876576874987222e-05, 1.2367996532702819e-05, 1.2143371350248344e-05, 1.1695758985297289e-05, 1.1424220247135963e-05, 1.1131935025332496e-05, 1.1190773875568993e-05, 1.0944204404950142e-05, 1.0946872862405144e-05, 1.0798782568599563e-05, 1.0945005669782404e-05, 1.076225635188166e-05, 1.0599945198919158e-05, 1.0455075425852556e-05, 1.450934632885037e-05, 1.4207863387127873e-05, 1.3732372281083371e-05, 1.2938263353134971e-05, 9.948860679287463e-06, 9.385978955833707e-06, 8.756065653869882e-06, 8.472309673379641e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 7.914906109363073e-05, "first_to_last_ratio": 1.7192461820598803, "imbalance_abs_log_ratio": 0.5418859285143861, "last_quartile_mean": 4.603707247952116e-05, "mean": 5.4592715059698094e-05, "normalized": [ 1.7241781198185933, 1.840889791100673, 1.5899852937072811, 1.4105138456143302, 1.3784873634586734, 1.2918312728835704, 1.2297133584866053, 1.1328804966903552, 1.0698731577046854, 0.991728141413645, 0.9453717076917117, 0.9099632530102919, 0.8959114507085134, 0.8626442255831185, 0.8400452309087872, 0.8167472645920701, 0.821645530449023, 0.8002665541585989, 0.7991850772986235, 0.7879704774609717, 0.7970876292631485, 0.7843957862785416, 0.7719533713478436, 0.7604728837369893, 1.0567363025522978, 1.035066582313382, 1.001583181765363, 0.9453853019538003, 0.7341845795920305, 0.6945525076290716, 0.6495623638855766, 0.6291878969418324 ], "peak_layer": 2, "peak_normalized": 1.840889791100673, "population_cv": 0.3029856274899433, "rest_mean": 5.626962943265594e-05, "spike_contrast": 0.8092709673914827, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.5537377445725724e-05, "top_five_layers": [ 2, 1, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 9.412756480742246e-05, 0.00010049917182186618, 8.680161408847198e-05, 7.700378046138212e-05, 7.525536784669384e-05, 7.052457658573985e-05, 6.713339098496363e-05, 6.184702215250582e-05, 5.8407280448591337e-05, 5.41411318408791e-05, 5.161040826351382e-05, 4.967736458638683e-05, 4.891023854725063e-05, 4.709409040515311e-05, 4.586034992826171e-05, 4.458845069166273e-05, 4.4855860323878005e-05, 4.368872396298684e-05, 4.362968320492655e-05, 4.3017447751481086e-05, 4.351517782197334e-05, 4.2822295654332265e-05, 4.2143030441366136e-05, 4.1516279452480376e-05, 5.7690103858476505e-05, 5.650709499605e-05, 5.467914525070228e-05, 5.1611150411190465e-05, 4.008112955489196e-05, 3.791750714299269e-05, 3.546137304510921e-05, 3.434907557675615e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.07019018055871129, "first_to_last_ratio": 1.7192461842102231, "imbalance_abs_log_ratio": 0.5418859297651335, "last_quartile_mean": 0.04082613717764616, "mean": 0.04841336654499173, "normalized": [ 1.7241780757691632, 1.8408898223226076, 1.5899853831381914, 1.410513823758526, 1.3784873270210494, 1.2918312963663738, 1.2297133800635207, 1.1328804963171313, 1.069873143198933, 0.9917281279670197, 0.9453716876252737, 0.9099632687949483, 0.8959114906104791, 0.8626442145108223, 0.8400452555516388, 0.8167472280339804, 0.8216454786918603, 0.8002665232543742, 0.7991851022834544, 0.7879704574858349, 0.7970875888178002, 0.784395781761838, 0.7719533617392295, 0.760472940078542, 1.05673627147129, 1.0350666081621565, 1.0015831863739189, 0.9453853073429236, 0.734184576587361, 0.694552506334395, 0.6495623468208583, 0.6291879417445048 ], "peak_layer": 2, "peak_normalized": 1.8408898223226076, "population_cv": 0.30298563129861156, "rest_mean": 0.04990047106036433, "spike_contrast": 0.8092709608518238, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.040383002161979674, "top_five_layers": [ 2, 1, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.08347326517105103, 0.0891236737370491, 0.07697654515504837, 0.06828772276639938, 0.06673721224069595, 0.0625419020652771, 0.05953456461429596, 0.05484655871987343, 0.05179616063833237, 0.04801289737224579, 0.045768626034259796, 0.044054385274648666, 0.043374091386795044, 0.041763510555028915, 0.04066941887140274, 0.03954148292541504, 0.03977862372994423, 0.03874359652400017, 0.038691241294145584, 0.03814830258488655, 0.03858969360589981, 0.037975240498781204, 0.03737286105751991, 0.03681705519556999, 0.05116016045212746, 0.050111059099435806, 0.0484900139272213, 0.045769285410642624, 0.035544347018003464, 0.03362562507390976, 0.03144749999046326, 0.030461106449365616 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 3.1499768908815895e-06, "first_to_last_ratio": 1.034206261098032, "imbalance_abs_log_ratio": 0.033634235011196216, "last_quartile_mean": 3.0457917432613613e-06, "mean": 2.767400765435468e-06, "normalized": [ 2.1814195138946744, 1.2474241889379907, 1.1222654250834863, 1.0031475000991907, 0.9117745127733886, 0.8816833725854125, 0.8770629409616562, 0.8811733142683152, 0.8629784955766571, 0.8316830327707667, 0.8235668776905775, 0.8250944234752862, 0.8171112056304486, 0.8225590852747408, 0.8353245106496434, 0.8506311898499032, 0.8463351316533214, 0.8685801814660532, 0.8873051880314596, 0.9074454256716082, 0.9316388522452362, 0.9651664774800727, 0.9968230398838435, 1.0170336898915424, 1.374498464971104, 1.3637737658333244, 1.3337903393137946, 1.2602746514195613, 0.9346785796224344, 0.8956270753967667, 0.8422654282672426, 0.7998641193304972 ], "peak_layer": 1, "peak_normalized": 2.1814195138946744, "population_cv": 0.27078047147820095, "rest_mean": 2.7381728597430523e-06, "spike_contrast": 1.0683151159598507, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.925231456174515e-06, "top_five_layers": [ 1, 25, 26, 27, 28 ], "top_five_spike_overlap": 1 }, "values": [ 6.0368620324879885e-06, 3.4521226552897133e-06, 3.105758196397801e-06, 2.7761111596191768e-06, 2.5232454845536267e-06, 2.4399712401645957e-06, 2.42718465415237e-06, 2.438559704387444e-06, 2.3882073492131894e-06, 2.3016002614895115e-06, 2.2791396077082027e-06, 2.2833669390820432e-06, 2.2612741759076016e-06, 2.276350642205216e-06, 2.311677690158831e-06, 2.354037405893905e-06, 2.3421484911523294e-06, 2.4037094590312336e-06, 2.4555290565331234e-06, 2.511265165594523e-06, 2.578218072812888e-06, 2.6710024485510075e-06, 2.7586088435782585e-06, 2.814539811879513e-06, 3.803788104050909e-06, 3.774108563447953e-06, 3.691132405947428e-06, 3.487685034997412e-06, 2.586630216683261e-06, 2.478559053997742e-06, 2.3308859908865998e-06, 2.2135445760795847e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 5.9450050230225315e-05, "first_to_last_ratio": 2.017444596891957, "imbalance_abs_log_ratio": 0.7018316595169721, "last_quartile_mean": 2.9467996455423418e-05, "mean": 3.904382947439444e-05, "normalized": [ 1.7889072073169312, 1.9066722375518106, 1.6645264384432352, 1.496663787464947, 1.4710728745130246, 1.3748895461916322, 1.2914944531447627, 1.1869659317343255, 1.113398940433113, 1.029594521735694, 0.9764204828260671, 0.9358568046936039, 0.9225664396829525, 0.8813934785026988, 0.8530914004121373, 0.8242402314819015, 0.835226334407852, 0.8028145075194778, 0.8023730358754295, 0.7855202425475775, 0.7905767607365202, 0.7659737081413304, 0.7434011642137429, 0.7184278736325621, 0.9558802179251441, 0.9293074079312249, 0.8990924102855112, 0.8518463721074153, 0.6531122929784916, 0.6161484536455182, 0.5764106480155923, 0.5561337939077728 ], "peak_layer": 2, "peak_normalized": 1.9066722375518106, "population_cv": 0.3438775168212192, "rest_mean": 4.052711941498004e-05, "spike_contrast": 0.765760415327251, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.1034063795232214e-05, "top_five_layers": [ 2, 1, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 6.984578794799745e-05, 7.444378570653498e-05, 6.49894864181988e-05, 5.843548569828272e-05, 5.743631845689379e-05, 5.3680952987633646e-05, 5.042488919571042e-05, 4.6343695430550724e-05, 4.347135836724192e-05, 4.019931293441914e-05, 3.812319482676685e-05, 3.6539433494908735e-05, 3.60205267497804e-05, 3.4412976674502715e-05, 3.3307955163763836e-05, 3.2181495043914765e-05, 3.261043457314372e-05, 3.134495273116045e-05, 3.1327715987572446e-05, 3.0669718398712575e-05, 3.086714423261583e-05, 2.9906546842539683e-05, 2.902522828662768e-05, 2.8050175387761556e-05, 3.732122422661632e-05, 3.628371996455826e-05, 3.5104010748909786e-05, 3.3259344490943477e-05, 2.5500004994682968e-05, 2.405679515504744e-05, 2.2505279048345983e-05, 2.1713593014283106e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 4.342638567322865e-05, "first_to_last_ratio": 2.386695182772136, "imbalance_abs_log_ratio": 0.8699096403219505, "last_quartile_mean": 1.8195195593762037e-05, "mean": 2.7162780583012136e-05, "normalized": [ 1.8784438713512777, 1.991275966230593, 1.7524296051911203, 1.5914241369986053, 1.5741899722006274, 1.4628631445632996, 1.3269551515206026, 1.212385818328642, 1.1369693713945288, 1.053121539457193, 0.9944120320787689, 0.9423727644416658, 0.92960571690996, 0.8844680573886359, 0.8583088993998432, 0.831099978563582, 0.8494188625886138, 0.8069477989199431, 0.8140135413007862, 0.7897755109318056, 0.7907931299819544, 0.75747286968027, 0.725223131761823, 0.6871681570811793, 0.8500462023663048, 0.8319121451758618, 0.8093900578538022, 0.7658449895571845, 0.5785345108955382, 0.5451076403327432, 0.4990735706636637, 0.478951854889581 ], "peak_layer": 2, "peak_normalized": 1.991275966230593, "population_cv": 0.3921722645830505, "rest_mean": 2.8359247254330614e-05, "spike_contrast": 0.729986250066319, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.070186055789236e-05, "top_five_layers": [ 2, 1, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 5.102375871501863e-05, 5.408859215094708e-05, 4.7600860852980986e-05, 4.322750464780256e-05, 4.2759376810863614e-05, 3.973543061874807e-05, 3.604379162425175e-05, 3.293176996521652e-05, 3.0883249564794824e-05, 2.860570930351969e-05, 2.7010995836462826e-05, 2.559746462793555e-05, 2.5250676117138937e-05, 2.4024611775530502e-05, 2.3314056306844577e-05, 2.2574986360268667e-05, 2.307257818756625e-05, 2.191894600400701e-05, 2.2110871213953942e-05, 2.145249891327694e-05, 2.1480140276253223e-05, 2.057506935670972e-05, 1.9699076801771298e-05, 1.866539787442889e-05, 2.3089618480298668e-05, 2.259704706375487e-05, 2.198528454755433e-05, 2.080247941194102e-05, 1.571460597915575e-05, 1.48066392284818e-05, 1.35562258947175e-05, 1.3009664144192357e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00016797798161860555, "first_to_last_ratio": 1.6975899913101038, "imbalance_abs_log_ratio": 0.529209593072091, "last_quartile_mean": 9.895085531752557e-05, "mean": 0.0001176193313767726, "normalized": [ 1.6707764727569818, 1.7774644420211134, 1.5539444173172348, 1.4132836817439416, 1.368907634940964, 1.2879938497023624, 1.2256078186986779, 1.1272171606279269, 1.0608955824824928, 0.9812630461027395, 0.9456653974174016, 0.9218162415513198, 0.9166741726954444, 0.877976399127856, 0.8517258914149402, 0.8288994705519163, 0.8347332603782261, 0.8190059801240409, 0.8187522913653438, 0.800771690768729, 0.8114248246876556, 0.8000953111326414, 0.7906441212716242, 0.7842164733415609, 1.0905486496890393, 1.0691456324684494, 1.038099968008601, 0.9857982662088418, 0.6939755898415391, 0.6518274388309239, 0.6079779462132741, 0.5928708765161964 ], "peak_layer": 2, "peak_normalized": 1.7774644420211134, "population_cv": 0.2932499751943919, "rest_mean": 0.00012076922333411251, "spike_contrast": 0.8330757789904475, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001006099148071371, "top_five_layers": [ 2, 1, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 0.00019651561160571873, 0.00020906417921651155, 0.00018277390336152166, 0.0001662294816924259, 0.00016101000073831528, 0.00015149297541938722, 0.00014415517216548324, 0.00013258252874948084, 0.0001247818290721625, 0.00011541550338733941, 0.00011122853175038472, 0.00010842340998351574, 0.00010781860328279436, 0.00010326699703000486, 0.0001001794298645109, 9.749460150487721e-05, 9.818076796364039e-05, 9.633093577576801e-05, 9.630109707359225e-05, 9.418623085366562e-05, 9.543924534227699e-05, 9.410667553311214e-05, 9.299503290094435e-05, 9.223901724908501e-05, 0.00012826960301026702, 0.0001257521944353357, 0.0001221006241394207, 0.00011594893294386566, 8.162494486896321e-05, 7.66675075283274e-05, 7.150995952542871e-05, 6.973307608859614e-05 ] } } }, "post_mlp_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "89f2226456bfe96e245d2aad0f699d32c676f9f53099789cd870253d002402f5", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.16915453970432281, 0.374446302652359, 0.39966297149658203, 0.4194087088108063, 0.09892481565475464, 0.11664740741252899, 0.13951337337493896, 0.18593810498714447, 0.06607726216316223, 0.10425819456577301, 0.14551182091236115, 0.19865906238555908, 0.06484340131282806, 0.10430748760700226, 0.14720691740512848, 0.20439592003822327, 0.10052042454481125, 0.14085541665554047, 0.1745898574590683, 0.2420409768819809, 0.049456026405096054, 0.08509506285190582, 0.11783231794834137, 0.15649692714214325, 0.05533855780959129, 0.08507220447063446, 0.11942370235919952, 0.1559232622385025, 0.06628894805908203, 0.12384890764951706, 0.19928614795207977, 0.30097496509552 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.9286424276288017e-05, "first_to_last_ratio": 1.7521203995902288, "imbalance_abs_log_ratio": 0.5608267113803039, "last_quartile_mean": 1.1007476587110432e-05, "mean": 1.329404926764255e-05, "normalized": [ 1.5830884964859404, 1.8642532193805597, 1.6087687990289774, 1.4071762895596276, 1.4140396857780653, 1.3278580276780159, 1.2500693586339857, 1.1507956501288648, 1.0468444989944645, 0.9815173799712515, 0.9385267769480207, 0.905437684407979, 0.8976442727572985, 0.864758300403981, 0.8426900344047217, 0.8244176324809771, 0.8360826421400467, 0.8172704563586889, 0.8192274978074702, 0.8139049826821911, 0.8167464760588842, 0.799833111692996, 0.7894070736006407, 0.7756393697890056, 1.0687281164979445, 1.0326907368420801, 0.9741867867992471, 0.9219037601983189, 0.7056642260146846, 0.6594293783384909, 0.6385485684123104, 0.6228507097242695 ], "peak_layer": 2, "peak_normalized": 1.8642532193805597, "population_cv": 0.29953527154923426, "rest_mean": 1.3663154041515318e-05, "spike_contrast": 0.8271064978402503, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.1300883488729597e-05, "top_five_layers": [ 2, 3, 1, 5, 4 ], "top_five_spike_overlap": 0 }, "values": [ 2.104565646732226e-05, 2.4783474145806395e-05, 2.138705167453736e-05, 1.8707070921664126e-05, 1.879831324913539e-05, 1.7652610040386207e-05, 1.6618483641650528e-05, 1.5298734069801867e-05, 1.3916802345192991e-05, 1.304834040638525e-05, 1.2476821211748756e-05, 1.2036933185299858e-05, 1.1933327186852694e-05, 1.149613945017336e-05, 1.1202762834727764e-05, 1.0959848623315338e-05, 1.1114923836430535e-05, 1.086483371182112e-05, 1.0890850717260037e-05, 1.0820092938956805e-05, 1.0857867891900241e-05, 1.0633020792738535e-05, 1.0494416528672446e-05, 1.0311387995898258e-05, 1.4207724234438501e-05, 1.3728641533816699e-05, 1.295088713959558e-05, 1.2255834008101374e-05, 9.381134987052064e-06, 8.766486644162796e-06, 8.488896128255874e-06, 8.280208021460567e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 7.694873238506261e-05, "first_to_last_ratio": 1.7432152215035543, "imbalance_abs_log_ratio": 0.5557312365419732, "last_quartile_mean": 4.414184286360978e-05, "mean": 5.3159041158323816e-05, "normalized": [ 1.5913407185814827, 1.8632489528458889, 1.6128734795733863, 1.4043006149694164, 1.4066272935698663, 1.3126652836956612, 1.2393507499620326, 1.1497464548522915, 1.049833614408579, 0.985390885890743, 0.9424718164101702, 0.9105508508722556, 0.9050202842384315, 0.8709194347820474, 0.8461280371574296, 0.8259390121005761, 0.8379023338495977, 0.8153953215324395, 0.8159707977472517, 0.8104068344678301, 0.8115020117618769, 0.7953124403225564, 0.7840013796989213, 0.7701158346270738, 1.0630434248930731, 1.0283170042534806, 0.9709691467241097, 0.9201848250266096, 0.7128611181523299, 0.6675462499646406, 0.6476462975123022, 0.6324174955556487 ], "peak_layer": 2, "peak_normalized": 1.8632489528458889, "population_cv": 0.2980021566402959, "rest_mean": 5.4686920419852766e-05, "spike_contrast": 0.8211925777002528, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.4908493146067484e-05, "top_five_layers": [ 2, 3, 1, 5, 4 ], "top_five_spike_overlap": 0 }, "values": [ 8.459414675598964e-05, 9.904852777253836e-05, 8.573880768381059e-05, 7.465127418981865e-05, 7.477495819330215e-05, 6.978002784308046e-05, 6.588269752683118e-05, 6.111941911512986e-05, 5.5808148317737505e-05, 5.238243466010317e-05, 5.010089807910845e-05, 4.840401015826501e-05, 4.81100105389487e-05, 4.629724207916297e-05, 4.497935515246354e-05, 4.3906125938519835e-05, 4.4542084651766345e-05, 4.3345633457647637e-05, 4.337622522143647e-05, 4.30804502684623e-05, 4.313866884331219e-05, 4.227804674883373e-05, 4.167676161159761e-05, 4.093861934961751e-05, 5.651036917697638e-05, 5.466434595291503e-05, 5.161578883416951e-05, 4.891614298685454e-05, 3.789501352002844e-05, 3.548611857695505e-05, 3.44282561854925e-05, 3.36187076754868e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.06823890749365091, "first_to_last_ratio": 1.743215282378554, "imbalance_abs_log_ratio": 0.5557312714630764, "last_quartile_mean": 0.03914542752318084, "mean": 0.047141969844233245, "normalized": [ 1.5913406918168036, 1.8632489685750107, 1.6128736136862545, 1.404300686062956, 1.4066272752430375, 1.3126652978642392, 1.2393507919180051, 1.1497464476397572, 1.0498335795335896, 0.9853908839486951, 0.9424717855432424, 0.9105508429124037, 0.9050202744406166, 0.870919408940349, 0.8461280610722746, 0.8259390008832959, 0.8379022620738577, 0.8153953052124332, 0.8159708282407102, 0.8104068331259242, 0.81150201006522, 0.7953124516622029, 0.7840013656127053, 0.7701158748927134, 1.063043425341901, 1.0283169363602562, 0.97096914453038, 0.9201848281477967, 0.7128610928945387, 0.6675462603619028, 0.6476462655632079, 0.632417505833719 ], "peak_layer": 2, "peak_normalized": 1.8632489685750107, "population_cv": 0.29800217008568586, "rest_mean": 0.04849690840476089, "spike_contrast": 0.8211925858241791, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.03982530161738396, "top_five_layers": [ 2, 3, 1, 5, 4 ], "top_five_spike_overlap": 0 }, "values": [ 0.07501893490552902, 0.08783722668886185, 0.07603403925895691, 0.06620150059461594, 0.06631118059158325, 0.06188162788748741, 0.05842543765902519, 0.054201312363147736, 0.04949122294783592, 0.046453267335891724, 0.044429976493120193, 0.0429251603782177, 0.04266443848609924, 0.041056856513023376, 0.03988814353942871, 0.03893639147281647, 0.039500363171100616, 0.03843934088945389, 0.038466472178697586, 0.03820417448878288, 0.038255803287029266, 0.03749259561300278, 0.036959368735551834, 0.0363047793507576, 0.05011396110057831, 0.04847688600420952, 0.045773398131132126, 0.04337932541966438, 0.033605676144361496, 0.031469445675611496, 0.030531320720911026, 0.029813406988978386 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 2.992641185528555e-06, "first_to_last_ratio": 1.0269553765947432, "imbalance_abs_log_ratio": 0.026598479753885757, "last_quartile_mean": 2.9140907713554043e-06, "mean": 2.6725970485586004e-06, "normalized": [ 2.0207188599051387, 1.2377901021367361, 1.1092854643979624, 0.9724317302628551, 0.9354869978511628, 0.9012437664734249, 0.8950883528937604, 0.8859566430310495, 0.8578432173784939, 0.8345394742684079, 0.8291732250016162, 0.8279893934563786, 0.8261758298882989, 0.833874861114876, 0.8457382739661629, 0.8616991136864209, 0.8674698989932849, 0.88828627718148, 0.9102804465239884, 0.9354855515603782, 0.954868995884654, 0.9876860994727537, 1.021625014254206, 1.036388835660307, 1.3981780500722671, 1.3627155104871023, 1.3104593229174804, 1.2283588399297363, 0.9133291424221982, 0.8643086476416543, 0.8394469090527371, 0.8060771522330267 ], "peak_layer": 1, "peak_normalized": 2.0207188599051387, "population_cv": 0.2456062284249433, "rest_mean": 2.6331270469705082e-06, "spike_contrast": 1.0959346076576224, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.8857350571342977e-06, "top_five_layers": [ 1, 25, 26, 27, 2 ], "top_five_spike_overlap": 1 }, "values": [ 5.400567260949174e-06, 3.3081141737056896e-06, 2.964673058158951e-06, 2.5989181722252397e-06, 2.5001797894219635e-06, 2.4086614303087117e-06, 2.392210490143043e-06, 2.3678051093156682e-06, 2.2926692508917768e-06, 2.230387735835393e-06, 2.2160459138831357e-06, 2.212882009189343e-06, 2.2080350845499197e-06, 2.2286114926828304e-06, 2.260317614855012e-06, 2.3029745079838904e-06, 2.3183974917628802e-06, 2.3740312826703303e-06, 2.4328128347406164e-06, 2.500175924069481e-06, 2.5519800601614406e-06, 2.6396869543532375e-06, 2.730391997829429e-06, 2.769849743344821e-06, 3.7367665299825603e-06, 3.641989451352856e-06, 3.5023297186853597e-06, 3.2829082101670792e-06, 2.4409607704001246e-06, 2.3099487407307606e-06, 2.243503331555985e-06, 2.1543194179685088e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 5.8193762924929615e-05, "first_to_last_ratio": 2.0502704916579684, "imbalance_abs_log_ratio": 0.7179717315962857, "last_quartile_mean": 2.8383456310621114e-05, "mean": 3.81904888513418e-05, "normalized": [ 1.6894591614399186, 1.9241519914027179, 1.6897173126933787, 1.4953035084539958, 1.4962677176744832, 1.3930853284779345, 1.2995553194328915, 1.2026702966805216, 1.0917791871758602, 1.021436780958654, 0.9712475096065674, 0.9342011853360711, 0.9289466165935991, 0.8873192030499892, 0.8563143801805745, 0.8310125090610249, 0.8486912975022995, 0.8159107559923885, 0.8169807977009178, 0.8077051659654394, 0.8024791748525016, 0.7739806100932451, 0.7515362161508304, 0.7245882735701051, 0.9569989892970601, 0.9222502111823565, 0.8748332556713925, 0.8323419403869606, 0.6311780047928108, 0.5936524336750243, 0.5743486259596642, 0.5600562389888216 ], "peak_layer": 2, "peak_normalized": 1.9241519914027179, "population_cv": 0.34212246721363443, "rest_mean": 3.9591396233407536e-05, "spike_contrast": 0.7735415242149172, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.0625588988186794e-05, "top_five_layers": [ 2, 3, 1, 5, 4 ], "top_five_spike_overlap": 0 }, "values": [ 6.452127126976848e-05, 7.348430517595261e-05, 6.45311301923357e-05, 5.7106371968984604e-05, 5.7143195590469986e-05, 5.3202609706204385e-05, 4.963065293850377e-05, 4.593056655721739e-05, 4.16955808759667e-05, 3.900916999555193e-05, 3.70924171875231e-05, 3.5677599953487515e-05, 3.547692540450953e-05, 3.38871541316621e-05, 3.270306478952989e-05, 3.1736773962620646e-05, 3.2411935535492375e-05, 3.116003063041717e-05, 3.120089604635723e-05, 3.084665513597429e-05, 3.064707198063843e-05, 2.9558697860920802e-05, 2.8701535484287888e-05, 2.76723803835921e-05, 3.654825923149474e-05, 3.52211864083074e-05, 3.341030969750136e-05, 3.178754559485242e-05, 2.4104996555252e-05, 2.267187664983794e-05, 2.1934654796496034e-05, 2.138882155122701e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 4.280252824173658e-05, "first_to_last_ratio": 2.451089090903835, "imbalance_abs_log_ratio": 0.8965324526833484, "last_quartile_mean": 1.746265706970007e-05, "mean": 2.6668096609228087e-05, "normalized": [ 1.8015320925519454, 1.9981655807622871, 1.7804161221172625, 1.6024982127415732, 1.601610138786345, 1.4821073059649263, 1.3412624599217569, 1.2324780385805596, 1.121305912410121, 1.0446267969277732, 0.9876705625973445, 0.9387097222017745, 0.9375932863723446, 0.8905309598377442, 0.8607759805802201, 0.8357113550501364, 0.8611571975798953, 0.8186175002062293, 0.8303829795272117, 0.815474250430593, 0.8016206377712574, 0.764117179150144, 0.7257517704082764, 0.6873677407608959, 0.8502799286310313, 0.8258737192215173, 0.7816710999562642, 0.7478205446125893, 0.5564676860136006, 0.5145078286317469, 0.490366472614564, 0.471528937080069 ], "peak_layer": 2, "peak_normalized": 1.9981655807622871, "population_cv": 0.3943725090628123, "rest_mean": 2.782456626677972e-05, "spike_contrast": 0.7339974417798157, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.0423160458449273e-05, "top_five_layers": [ 2, 1, 3, 4, 5 ], "top_five_spike_overlap": 0 }, "values": [ 4.8043431888800114e-05, 5.328727274900302e-05, 4.748030914925039e-05, 4.273557715350762e-05, 4.271189391147345e-05, 3.952498082071543e-05, 3.5768916859524325e-05, 3.28678434016183e-05, 2.9903094400651753e-05, 2.7858208341058344e-05, 2.633929398143664e-05, 2.5033601559698582e-05, 2.5003828341141343e-05, 2.374876567046158e-05, 2.295525700901635e-05, 2.228683115390595e-05, 2.2965423340792768e-05, 2.1830970581504516e-05, 2.2144733520690352e-05, 2.1747146092820913e-05, 2.1377696612034924e-05, 2.0377550754346885e-05, 1.9354418327566236e-05, 1.833078931667842e-05, 2.2675347281619906e-05, 2.2024480131221935e-05, 2.0845680410275236e-05, 1.9942950530094095e-05, 1.4839934010524303e-05, 1.3720944480155595e-05, 1.3077140465611592e-05, 1.2574779248097911e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.0001629189446248347, "first_to_last_ratio": 1.714305633781884, "imbalance_abs_log_ratio": 0.5390081203712778, "last_quartile_mean": 9.503494675300317e-05, "mean": 0.00011443965263424616, "normalized": [ 1.556137633136766, 1.785096132874448, 1.5836711508630803, 1.3908688787327412, 1.402502180982902, 1.3076898838961202, 1.239896369026688, 1.1231233799092157, 1.0322172488550143, 0.9795330268170321, 0.9467492196150763, 0.915697240541664, 0.921096050539655, 0.8878475444944404, 0.8588609925139414, 0.8367374111089961, 0.8542051666016423, 0.8355631071501797, 0.8376251009775936, 0.8222878138853794, 0.8251706124831407, 0.8118731937295658, 0.8031306374510078, 0.7989222804754791, 1.0985923164412201, 1.067525841357921, 1.0136065974249735, 0.960683699855784, 0.6713099635441048, 0.6233626306316047, 0.6123441372218205, 0.5960725568608024 ], "peak_layer": 2, "peak_normalized": 1.785096132874448, "population_cv": 0.28809788917989526, "rest_mean": 0.00011724686137986955, "spike_contrast": 0.8467665934887498, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.92807254078798e-05, "top_five_layers": [ 2, 3, 1, 5, 4 ], "top_five_spike_overlap": 0 }, "values": [ 0.00017808385018724948, 0.00020428578136488795, 0.00018123477639164776, 0.00015917055134195834, 0.00016050186241045594, 0.00014965157606638968, 0.00014189330977387726, 0.00012852984946221113, 0.00011812658340204507, 0.00011209741933271289, 0.00010834565182449296, 0.00010479207412572578, 0.00010540991206653416, 0.00010160496458411217, 9.828775364439934e-05, 9.575593867339194e-05, 9.775494254427031e-05, 9.562155173625797e-05, 9.585752559360117e-05, 9.410233178641647e-05, 9.443223825655878e-05, 9.291048627346754e-05, 9.190999116981402e-05, 9.14283882593736e-05, 0.00012572252308018506, 0.00012216728646308184, 0.00011599678691709414, 0.00010994030890287831, 7.68244790378958e-05, 7.133740291465074e-05, 7.00764503562823e-05, 6.821433635195717e-05 ] } } }, "pre_attention_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "ee695e78937de01b7128d3866b8fc5dd328ebdf2c159be37918d4f02952d5248", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.0626930519938469, 0.09030644595623016, 0.20229491591453552, 0.20300470292568207, 0.19049881398677826, 0.12695962190628052, 0.12132706493139267, 0.11585728824138641, 0.13427142798900604, 0.10485932976007462, 0.10419609397649765, 0.10852954536676407, 0.1286265105009079, 0.09096142649650574, 0.10080574452877045, 0.10566592961549759, 0.1287592649459839, 0.09650235623121262, 0.1065765842795372, 0.11446042358875275, 0.12395420670509338, 0.07221997529268265, 0.07677769660949707, 0.08416415005922318, 0.09094838052988052, 0.059439923614263535, 0.062182098627090454, 0.06896103918552399, 0.0776243805885315, 0.06285123527050018, 0.07231374084949493, 0.08016443997621536 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.4809331275955628e-05, "first_to_last_ratio": 3.613243953818226, "imbalance_abs_log_ratio": 1.2846059710092836, "last_quartile_mean": 4.098624799553363e-06, "mean": 8.899090417457955e-06, "normalized": [ 2.7947820845857536, 2.1848011520247117, 1.6751146254914047, 1.3815075976990288, 0.5644309797720112, 1.25688764936562, 1.6848127717835277, 1.7707821941450765, 1.1415279166354348, 1.09837258111741, 0.8439171701178217, 0.7432638346666525, 0.5134480292182424, 1.0343165563428867, 0.8019756742245486, 0.7891310189278757, 0.6878522653085101, 1.2023808642289078, 1.136157261395838, 1.29198362939584, 0.582318378846126, 1.1488975185131152, 0.9860408554935766, 1.0007637071111952, 0.467244042019967, 0.5943828838712202, 0.4449259290238304, 0.5908637505278774, 0.3896376394541326, 0.27514441672297546, 0.4300077204606269, 0.4923273015082546 ], "peak_layer": 1, "peak_normalized": 2.7947820845857536, "population_cv": 0.5542485862815736, "rest_mean": 9.167623930908223e-06, "spike_contrast": 0.8125343601532914, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 7.449009444826515e-06, "top_five_layers": [ 1, 2, 8, 7, 3 ], "top_five_spike_overlap": 0 }, "values": [ 2.487101846782025e-05, 1.9442742996034212e-05, 1.4906996511854231e-05, 1.2294161024328787e-05, 5.02292232340551e-06, 1.1185156836290844e-05, 1.499330119258957e-05, 1.5758350855321623e-05, 1.0158560144191142e-05, 9.774516911420505e-06, 7.510095201723743e-06, 6.614372068725061e-06, 4.569220436678734e-06, 9.204476555169094e-06, 7.136854037526064e-06, 7.022548288659891e-06, 6.121259502833709e-06, 1.0700096026994288e-05, 1.0110766197612975e-05, 1.1497479135869071e-05, 5.182103905099211e-06, 1.0224142897641286e-05, 8.774866728344932e-06, 8.905886716092937e-06, 4.1580469769542105e-06, 5.2894670261594e-06, 3.959436071454547e-06, 5.258149940345902e-06, 3.4674205835472094e-06, 2.4485350422764895e-06, 3.826677584584104e-06, 4.381265171105042e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 5.9309133803253644e-05, "first_to_last_ratio": 3.674522433940219, "imbalance_abs_log_ratio": 1.3014231742723341, "last_quartile_mean": 1.6140637285388948e-05, "mean": 3.5482112963336476e-05, "normalized": [ 2.9256456140919034, 2.146447006262613, 1.618600231422083, 1.4241008383852924, 0.5633010631437305, 1.2904087725607365, 1.65936459642797, 1.7443081885217793, 1.103852168684191, 1.0728671090702615, 0.864947264988498, 0.7335332605693856, 0.518662194348195, 1.0301047431818724, 0.8131320183118628, 0.7747243551731057, 0.686256501554472, 1.2797425832136473, 1.0924417145243503, 1.290566566133794, 0.6082802800651689, 1.1286554933359485, 0.9968233205687433, 0.9940737752479619, 0.44886350430992, 0.6021005957017376, 0.4321734248386855, 0.5410138744111522, 0.38802834021978727, 0.2878017638777764, 0.434380074143002, 0.5047987627103729 ], "peak_layer": 1, "peak_normalized": 2.9256456140919034, "population_cv": 0.5633089305021575, "rest_mean": 3.656405934398882e-05, "spike_contrast": 0.8106212231243035, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.9639602507813835e-05, "top_five_layers": [ 1, 2, 8, 7, 3 ], "top_five_spike_overlap": 0 }, "values": [ 0.00010380808816989884, 7.616047514602542e-05, 5.7431356253800914e-05, 5.053010681876913e-05, 1.998711195483338e-05, 4.5786429836880416e-05, 5.887776205781847e-05, 6.189174018800259e-05, 3.916700734407641e-05, 3.806759195867926e-05, 3.069015656365082e-05, 2.6027310013887472e-05, 1.8403230569674633e-05, 3.6550292861647904e-05, 2.88516421278473e-05, 2.7488857085700147e-05, 2.434983070997987e-05, 4.5407970901578665e-05, 3.876214032061398e-05, 4.579202868626453e-05, 2.158306961064227e-05, 4.004708171123639e-05, 3.536939766490832e-05, 3.527183798723854e-05, 1.592662556504365e-05, 2.1363801351981238e-05, 1.5334426279878244e-05, 1.9196315406588838e-05, 1.376806540065445e-05, 1.0211814696958754e-05, 1.541272285976447e-05, 1.7911326722241938e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.052595932967960835, "first_to_last_ratio": 3.6745224510047136, "imbalance_abs_log_ratio": 1.3014231789163375, "last_quartile_mean": 0.014313678489997983, "mean": 0.031465892563574016, "normalized": [ 2.9256457580801216, 2.146447001384909, 1.618600238124273, 1.4241008486741455, 0.5633010732744238, 1.290408690320605, 1.6593646381070337, 1.744308082121541, 1.1038521782453636, 1.0728670242611564, 0.864947262062369, 0.7335332497497623, 0.5186621971406641, 1.030104769818697, 0.813132033513683, 0.7747243322494269, 0.6862564953809476, 1.2797425745290079, 1.0924417358766922, 1.2905665060239992, 0.6082802654167409, 1.1286555265636042, 0.9968234151088091, 0.9940737754153864, 0.44886352585520944, 0.6021005907955316, 0.4321734496988843, 0.5410138376616499, 0.3880283543940306, 0.2878017677304838, 0.43438005775123895, 0.5047987446696087 ], "peak_layer": 1, "peak_normalized": 2.9256457580801216, "population_cv": 0.5633089412302545, "rest_mean": 0.03242537327524689, "spike_contrast": 0.8106212532210361, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.026284696720540523, "top_five_layers": [ 1, 2, 8, 7, 3 ], "top_five_spike_overlap": 0 }, "values": [ 0.09205805510282516, 0.06753987073898315, 0.050930701196193695, 0.04481060430407524, 0.017724771052598953, 0.04060386121273041, 0.0522133894264698, 0.05488621070981026, 0.034733694046735764, 0.03375871852040291, 0.027216337621212006, 0.023081278428435326, 0.01632016897201538, 0.032413166016340256, 0.025585925206542015, 0.02437739260494709, 0.021593673154711723, 0.04026824235916138, 0.0343746542930603, 0.04060882702469826, 0.019140081480145454, 0.03551415354013443, 0.03136593848466873, 0.031279418617486954, 0.014123891480267048, 0.018945632502436638, 0.013598723337054253, 0.01702348329126835, 0.01220965851098299, 0.009055939503014088, 0.01366815622895956, 0.01588394306600094 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 6.141809862469927e-06, "first_to_last_ratio": 2.1233361143593097, "imbalance_abs_log_ratio": 0.7529884906604624, "last_quartile_mean": 2.8925283288572246e-06, "mean": 3.2888173926437503e-06, "normalized": [ 9.776488697526496, 2.3730747752970984, 0.4010632508898084, 0.4725380103920625, 0.26376876487223105, 0.5052795981087146, 0.5072914034109365, 0.6403577763383237, 0.5052741018451524, 0.5812380988083414, 0.4912459088497901, 0.5233772384725672, 0.37717500303782525, 0.6574221882878839, 0.597678632992055, 0.6007687084156229, 0.43859630142792266, 0.8492341125783786, 0.7093873207739397, 0.8438638136981231, 0.3791926502805629, 0.7442638439236838, 0.8023334931394289, 0.9230536334245564, 0.6727019392614287, 1.0141980683725227, 1.0315103998332005, 1.293827079526469, 0.7442489106792886, 0.7566802144198279, 0.7225914890473246, 0.8002745720684321 ], "peak_layer": 1, "peak_normalized": 9.776488697526496, "population_cv": 1.61844059217288, "rest_mean": 3.4689050473254476e-06, "spike_contrast": 0.6677450162980114, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.316344057362585e-06, "top_five_layers": [ 1, 2, 28, 27, 26 ], "top_five_spike_overlap": 0 }, "values": [ 3.2153086067410186e-05, 7.804609595041256e-06, 1.319023795076646e-06, 1.5540912272626883e-06, 8.674873015479534e-07, 1.6617723304079846e-06, 1.6683887906765449e-06, 2.1060197923361557e-06, 1.6617542542007868e-06, 1.91158596862806e-06, 1.615618089090276e-06, 1.7212921648024349e-06, 1.240459710061259e-06, 2.162141527151107e-06, 1.9656558833958115e-06, 1.9758185771934222e-06, 1.442463144485373e-06, 2.792975919874152e-06, 2.333045358682284e-06, 2.775313987513073e-06, 1.2470953834053944e-06, 2.447747874612105e-06, 2.638728346937569e-06, 3.0357548439496895e-06, 2.2123938379081665e-06, 3.335512246849248e-06, 3.392449343664339e-06, 4.25516100222012e-06, 2.4476987618982093e-06, 2.4885830498533323e-06, 2.376471456955187e-06, 2.631956931509194e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 3.7269868016664986e-05, "first_to_last_ratio": 3.65169794831635, "imbalance_abs_log_ratio": 1.2951922507460107, "last_quartile_mean": 1.020617492031306e-05, "mean": 2.3355898292720667e-05, "normalized": [ 2.064236419340838, 2.207065559672527, 1.6207325352459208, 1.4258504453553775, 0.6578775786023344, 1.4081768256053158, 1.7689713156773732, 1.6129842733074633, 1.1848807295950294, 1.1209138771530585, 0.9203334409167135, 0.7870468714532032, 0.5753736356388351, 1.090780639746211, 0.8670438130517244, 0.7842104318846259, 0.7150809790549051, 1.3267599508323216, 1.1418619413078777, 1.2895246332305776, 0.6122856676653687, 1.192915828626597, 1.0963324912312633, 1.0328809019465057, 0.4791721506104629, 0.6443185874982661, 0.44398129690533406, 0.4848176933860423, 0.38963606492413133, 0.30082338036170775, 0.3361558224011495, 0.41697421777093835 ], "peak_layer": 2, "peak_normalized": 2.207065559672527, "population_cv": 0.49574488151536245, "rest_mean": 2.3863165013069562e-05, "spike_contrast": 0.8639532095405257, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.061665800283663e-05, "top_five_layers": [ 2, 1, 7, 3, 8 ], "top_five_spike_overlap": 0 }, "values": [ 4.82120958622545e-05, 5.154799873707816e-05, 3.785366425290704e-05, 3.330201798235066e-05, 1.536532181489747e-05, 3.2889234717004e-05, 4.131591413170099e-05, 3.767269663512707e-05, 2.7673953809426166e-05, 2.6179950509686023e-05, 2.1495214241440408e-05, 1.838218668126501e-05, 1.3438368114293553e-05, 2.547616168158129e-05, 2.0250587112968788e-05, 1.831593908718787e-05, 1.670135861786548e-05, 3.098767047049478e-05, 2.6669211365515366e-05, 3.0118006179691292e-05, 1.4300481780082919e-05, 2.7861620765179396e-05, 2.560583016020246e-05, 2.4123861294356175e-05, 1.1191496014362201e-05, 1.5048639397718944e-05, 1.0369582014391199e-05, 1.1323352737235837e-05, 9.100300303543918e-06, 7.02600027580047e-06, 7.851221198507119e-06, 9.738807420944795e-06 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 2.325335424302466e-05, "first_to_last_ratio": 3.533524419116561, "imbalance_abs_log_ratio": 1.2622957918696391, "last_quartile_mean": 6.580782098808413e-06, "mean": 1.5276127143692975e-05, "normalized": [ 1.3651111311078925, 2.2645962861550135, 1.602146720954942, 1.4236632649048078, 0.7573330188490042, 1.4889516524373358, 1.8740093094072992, 1.401806162460925, 1.2680810711127326, 1.217394607842109, 0.97732763236964, 0.8350505280195131, 0.633723858281215, 1.1496014989398466, 0.8965135445644448, 0.7794067764168341, 0.7400290080797218, 1.3550010349529762, 1.2099293831426021, 1.2473383821368482, 0.5885564148882477, 1.2326791832859807, 1.2095364389771577, 1.0359039381864297, 0.500505605369451, 0.6968801664942383, 0.46193045525875637, 0.43272303451526195, 0.3857898727000723, 0.31567788715723233, 0.2988002208074572, 0.3540019102240112 ], "peak_layer": 2, "peak_normalized": 2.2645962861550135, "population_cv": 0.4722076988678451, "rest_mean": 1.5521008225294753e-05, "spike_contrast": 0.8990246703369931, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.3953769303043373e-05, "top_five_layers": [ 2, 7, 3, 6, 4 ], "top_five_spike_overlap": 0 }, "values": [ 2.0853611204074696e-05, 3.45942607964389e-05, 2.4474597012158483e-05, 2.1748061044490896e-05, 1.1569115486054216e-05, 2.2745414753444493e-05, 2.862760447897017e-05, 2.1414169168565422e-05, 1.9371367670828477e-05, 1.8597074813442305e-05, 1.4929781173123047e-05, 1.2756338037434034e-05, 9.680846233095508e-06, 1.756145866238512e-05, 1.3695254892809317e-05, 1.1906317013199441e-05, 1.1304777217446826e-05, 2.069916808977723e-05, 1.84830350917764e-05, 1.905449971673079e-05, 8.990862625068985e-06, 1.8830563931260258e-05, 1.84770324267447e-05, 1.5824600268388167e-05, 7.645787263754755e-06, 1.0645630027283914e-05, 7.056508366076741e-06, 6.610332093259785e-06, 5.8933751461154316e-06, 4.822335540666245e-06, 4.564510163618252e-06, 5.407778189692181e-06 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00011627300682448549, "first_to_last_ratio": 3.8096405700640426, "imbalance_abs_log_ratio": 1.3375348461225791, "last_quartile_mean": 3.052072884202062e-05, "mean": 7.040946042025098e-05, "normalized": [ 2.763443313037709, 2.173300242626541, 1.628401967545918, 1.3602188204057024, 0.5681837913059802, 1.302237229843012, 1.6578289516136302, 1.7574520523157187, 1.1019692244304016, 1.0002282158447329, 0.8768116937088071, 0.7646096462396708, 0.5188493515037768, 1.0425647759194792, 0.8280656592488386, 0.7787973556284107, 0.6781993146029439, 1.2797059761032488, 1.09578239130086, 1.4072917644734235, 0.660469030971171, 1.166277980833339, 1.054547515341598, 1.0669651001588918, 0.45695885425368676, 0.6018979500195134, 0.43230220064100155, 0.5119990381319063, 0.3916082204777694, 0.2747082565057612, 0.34316757500403905, 0.455156539962517 ], "peak_layer": 1, "peak_normalized": 2.763443313037709, "population_cv": 0.5543466577062031, "rest_mean": 7.196050656030248e-05, "spike_contrast": 0.8620535656178147, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.203381126397289e-05, "top_five_layers": [ 1, 2, 8, 7, 3 ], "top_five_spike_overlap": 0 }, "values": [ 0.00019457255257293582, 0.00015302089741453528, 0.00011465490388218313, 9.577227319823578e-05, 4.000551416538656e-05, 9.168982069240883e-05, 0.00011672684195218608, 0.00012374125071801245, 7.758905849186704e-05, 7.042552897473797e-05, 6.173583824420348e-05, 5.3835752623854205e-05, 3.6531902878778055e-05, 7.34064233256504e-05, 5.8303656260250136e-05, 5.483470158651471e-05, 4.7751647798577324e-05, 9.010340727400035e-05, 7.715344690950587e-05, 9.908665379043669e-05, 4.650326809496619e-05, 8.211700333049521e-05, 7.425012154271826e-05, 7.512443698942661e-05, 3.2174226362258196e-05, 4.237930988892913e-05, 3.0438164685619995e-05, 3.604957601055503e-05, 2.7572923499974422e-05, 1.9342060113558546e-05, 2.4162243789760396e-05, 3.204732638550922e-05 ] } } }, "pre_mlp_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "e53fa425d89dfeb0e9395eca9bcd73dc0a55314abf45c5b10b43e2cbfa724345", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.059514161199331284, 0.08157797157764435, 0.14292451739311218, 0.10997853428125381, 0.0991024449467659, 0.1065012738108635, 0.12155740708112717, 0.1095810979604721, 0.06964563578367233, 0.08311934769153595, 0.09363401681184769, 0.0941167026758194, 0.08564268052577972, 0.09493856877088547, 0.09063401818275452, 0.09473779052495956, 0.10505300015211105, 0.09355910867452621, 0.09618847072124481, 0.11416846513748169, 0.047205954790115356, 0.053352247923612595, 0.06512638926506042, 0.07829347252845764, 0.05368548259139061, 0.05310077592730522, 0.061235878616571426, 0.07320034503936768, 0.05304349213838577, 0.0639304369688034, 0.07404901832342148, 0.08157160878181458 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 6.54528332688642e-06, "first_to_last_ratio": 0.286629321479513, "imbalance_abs_log_ratio": 1.2495654607977518, "last_quartile_mean": 2.28353585498553e-05, "mean": 1.1082081641689001e-05, "normalized": [ 1.459847898964845, 0.499289913877982, 0.3857396146807835, 0.5772848293172743, 0.33431101036601507, 0.2738230003356312, 0.3705820558046023, 0.8240703197049958, 0.7888129282709969, 0.5977191748776386, 0.5454072010017417, 0.550464658441738, 0.48142260271571663, 0.5131912434779171, 0.6104412558677625, 0.6389678041596253, 0.4251056645306276, 0.5426844818230171, 0.5145980282259913, 0.3979984575347756, 0.7237213430775018, 1.0362037591657798, 1.168325644140394, 1.2554606185468358, 1.849132071197145, 2.458148281977514, 3.025535514969934, 2.213045286944357, 2.0953557996853496, 2.094620133700358, 1.411652749364399, 1.3370366532507543 ], "peak_layer": 27, "peak_normalized": 3.025535514969934, "population_cv": 0.712179309611278, "rest_mean": 1.065815366827236e-05, "spike_contrast": 1.2545599467141373, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.337129269813886e-05, "top_five_layers": [ 27, 26, 28, 29, 30 ], "top_five_spike_overlap": 0 }, "values": [ 1.6178153600776568e-05, 5.533171588467667e-06, 4.2747979023260996e-06, 6.397517609002534e-06, 3.704861910591717e-06, 3.0345288450916996e-06, 4.106820597371552e-06, 9.13241456146352e-06, 8.741689271118958e-06, 6.623972694796976e-06, 6.044247129466385e-06, 6.1002942857157905e-06, 5.335164587449981e-06, 5.687227258022176e-06, 6.76495983498171e-06, 7.081093372107716e-06, 4.711055680672871e-06, 6.014073733240366e-06, 5.702817361452617e-06, 4.410651399666676e-06, 8.02033900981769e-06, 1.148329465650022e-05, 1.2947480172442738e-05, 1.3913117072661407e-05, 2.049223257927224e-05, 2.7241399948252365e-05, 3.352923158672638e-05, 2.4525148546672426e-05, 2.322090404049959e-05, 2.3212751329992898e-05, 1.5644051018171012e-05, 1.4817149349255487e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 2.6258697516823304e-05, "first_to_last_ratio": 0.2877403339449016, "imbalance_abs_log_ratio": 1.2456968237952706, "last_quartile_mean": 9.125831320488942e-05, "mean": 4.4300615684278455e-05, "normalized": [ 1.4581504581606346, 0.5123822794128472, 0.38589869979133096, 0.5804517355057356, 0.32871531284592487, 0.2736377361496381, 0.3717010073154115, 0.8309738456376156, 0.7859004073798295, 0.5927247340022558, 0.540237805667275, 0.5510681151470448, 0.48159538777010413, 0.5167264009771451, 0.6095111860731227, 0.6355663895068415, 0.4253831552140474, 0.5470620827609095, 0.5181460963164882, 0.39864766741018065, 0.7291029857337589, 1.0315755324345433, 1.163437866983406, 1.2515755888152456, 1.841832156848527, 2.452760369352008, 3.0363469642911127, 2.217531271943396, 2.102109745221804, 2.0822361455981877, 1.4163729171983446, 1.3306379525352834 ], "peak_layer": 27, "peak_normalized": 3.0363469642911127, "population_cv": 0.7120607310063193, "rest_mean": 4.263109918543638e-05, "spike_contrast": 1.250636408554989, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.3316004778025675e-05, "top_five_layers": [ 27, 26, 28, 29, 30 ], "top_five_spike_overlap": 0 }, "values": [ 6.459696305682883e-05, 2.2698850443703122e-05, 1.70955499925185e-05, 2.571436925791204e-05, 1.4562290743924677e-05, 1.2122320185881108e-05, 1.646658347453922e-05, 3.681265297927894e-05, 3.48158719134517e-05, 2.6258070647600107e-05, 2.393286740698386e-05, 2.441265678498894e-05, 2.133497218892444e-05, 2.289129770360887e-05, 2.7001720809494145e-05, 2.815598236338701e-05, 1.8844735677703284e-05, 2.4235187083831988e-05, 2.2954191081225872e-05, 1.766033710737247e-05, 3.229971116525121e-05, 4.5699431211687624e-05, 5.1541013817768544e-05, 5.544556915992871e-05, 8.159429853549227e-05, 0.00010865879448829219, 0.00013451203994918615, 9.823800064623356e-05, 9.312475594924763e-05, 9.224434325005859e-05, 6.274619227042422e-05, 5.894808055018075e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.023286475334316492, "first_to_last_ratio": 0.2877403327785342, "imbalance_abs_log_ratio": 1.245696827848812, "last_quartile_mean": 0.08092878432944417, "mean": 0.03928622876992449, "normalized": [ 1.4581505052971455, 0.5123823157657251, 0.38589867930614924, 0.580451723577461, 0.3287153178663167, 0.2736377440159473, 0.37170101175974146, 0.8309737615139164, 0.7859003914779473, 0.5927247417096759, 0.5402378159511653, 0.5510681293234533, 0.48159536884849885, 0.5167264079450129, 0.6095112082735188, 0.6355664211527974, 0.425383174484481, 0.54706208587854, 0.518146117805056, 0.3986476444537944, 0.7291029887649275, 1.0315754838162123, 1.1634378170072712, 1.2515756088361643, 1.84183212066326, 2.452760431127763, 3.0363469047992817, 2.217531234330326, 2.102109733022382, 2.082236260414343, 1.4163728970665859, 1.3306379537451396 ], "peak_layer": 27, "peak_normalized": 3.0363469047992817, "population_cv": 0.7120607301224652, "rest_mean": 0.03780568501463643, "spike_contrast": 1.2506363799564848, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.04728116504848003, "top_five_layers": [ 27, 26, 28, 29, 30 ], "top_five_spike_overlap": 0 }, "values": [ 0.057285234332084656, 0.020129568874835968, 0.015160503797233105, 0.022803759202361107, 0.012913985177874565, 0.010750195011496544, 0.014602730982005596, 0.032645825296640396, 0.030875062569975853, 0.023285919800400734, 0.02122390642762184, 0.02164938859641552, 0.018920065835118294, 0.020300231873989105, 0.02394539676606655, 0.024969007819890976, 0.016711700707674026, 0.0214920062571764, 0.020356006920337677, 0.015661362558603287, 0.028643706813454628, 0.04052671045064926, 0.045707084238529205, 0.04916968569159508, 0.072358638048172, 0.09635970741510391, 0.11928661912679672, 0.08711843937635422, 0.0825839638710022, 0.08180321007966995, 0.055643949657678604, 0.05227574706077576 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 1.5894347100697814e-06, "first_to_last_ratio": 0.2601635109042347, "imbalance_abs_log_ratio": 1.3464449575406465, "last_quartile_mean": 6.109368314355379e-06, "mean": 2.884904601430094e-06, "normalized": [ 1.625855142215518, 0.46098167799982964, 0.37367171303518437, 0.566555869913108, 0.3025369053707995, 0.2499972490188184, 0.2966999079504548, 0.531291884019451, 0.6616877085534597, 0.5252151708617063, 0.4734609648171375, 0.5201566289200071, 0.47957089848558887, 0.5627635299044025, 0.6253525089675875, 0.6462632244542076, 0.4361249333090643, 0.5648455062052933, 0.5106024021592923, 0.3759629036793034, 0.7400233279140302, 1.0619373276383908, 1.2027050448603398, 1.2641213753623195, 1.8450777258507045, 2.444384408644655, 3.3006111914510363, 2.322400269629467, 2.0934047870936956, 2.070888295367132, 1.483205875166526, 1.381643641181488 ], "peak_layer": 27, "peak_normalized": 3.3006111914510363, "population_cv": 0.7599857725159902, "rest_mean": 2.765890020260418e-06, "spike_contrast": 1.2753881441078445, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.527583339746343e-06, "top_five_layers": [ 27, 26, 28, 29, 30 ], "top_five_spike_overlap": 0 }, "values": [ 4.690436981036328e-06, 1.3298881640366744e-06, 1.078007244359469e-06, 1.634459636079555e-06, 8.727901104066405e-07, 7.212182140392542e-07, 8.559509296901524e-07, 1.5327264009101782e-06, 1.9089059151156107e-06, 1.5151956631598296e-06, 1.3658897159984917e-06, 1.5006022522356943e-06, 1.3835162917530397e-06, 1.623519096938253e-06, 1.8040823306364473e-06, 1.864407749962993e-06, 1.2581788269017125e-06, 1.6295253999487613e-06, 1.4730392194906017e-06, 1.0846171107914415e-06, 2.134896703864797e-06, 3.063587882934371e-06, 3.469689318080782e-06, 3.646869572548894e-06, 5.322873221302871e-06, 7.051815828162944e-06, 9.52194841374876e-06, 6.699903224216541e-06, 6.0392731029423885e-06, 5.9743151723523624e-06, 4.2789074541360606e-06, 3.985910097981105e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 1.65944162517917e-05, "first_to_last_ratio": 0.307005678217865, "imbalance_abs_log_ratio": 1.180889035742321, "last_quartile_mean": 5.405247338785557e-05, "mean": 2.780792223688877e-05, "normalized": [ 1.4983464744297323, 0.5348157127803579, 0.2733254102077556, 0.5269415035870197, 0.3698159771853738, 0.30184070647613875, 0.3920330181138424, 0.8768923183464846, 0.878557069848804, 0.6561735462067678, 0.598145411021264, 0.6321924901058512, 0.5462797999628135, 0.5755350112260291, 0.6729191147663853, 0.7039366077672822, 0.45782268923285163, 0.5922850932573034, 0.5483899460812152, 0.4376986236066998, 0.7694581525048373, 1.0894035058102869, 1.2298451477649552, 1.2871094596383732, 1.866623521311793, 2.3588104100725813, 2.771348522398077, 2.048018246461417, 2.037990489867878, 1.9483974233583767, 1.311932899623253, 1.2071156969781989 ], "peak_layer": 27, "peak_normalized": 2.771348522398077, "population_cv": 0.6524844885570633, "rest_mean": 2.652830486719419e-05, "spike_contrast": 1.3087099310356916, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.471785603323951e-05, "top_five_layers": [ 27, 26, 28, 29, 30 ], "top_five_spike_overlap": 0 }, "values": [ 4.1665902244858444e-05, 1.4872113752062432e-05, 7.600611752422992e-06, 1.4653148355137091e-05, 1.0283813935529906e-05, 8.393562893616036e-06, 1.0901623682002537e-05, 2.4384553398704156e-05, 2.44308466790244e-05, 1.824682294682134e-05, 1.6633181076031178e-05, 1.7579959603608586e-05, 1.5190906196949072e-05, 1.6004432836780325e-05, 1.8712482415139675e-05, 1.9575014448491856e-05, 1.2731097740470432e-05, 1.6470217815367505e-05, 1.5249584976118058e-05, 1.2171489288448356e-05, 2.1397032469394617e-05, 3.029404797416646e-05, 3.419943823246285e-05, 3.579183976398781e-05, 5.190692172618583e-05, 6.559361645486206e-05, 7.706544420216233e-05, 5.6951132137328386e-05, 5.6672281061764807e-05, 5.418088403530419e-05, 3.6482128052739426e-05, 3.3567379432497546e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 1.0420098931263055e-05, "first_to_last_ratio": 0.3625514134753626, "imbalance_abs_log_ratio": 1.014588984467508, "last_quartile_mean": 2.874102415262314e-05, "mean": 1.6328994817627063e-05, "normalized": [ 1.6215030774368884, 0.5678405711021709, 0.2373409582908636, 0.5009488827160783, 0.42742198585310837, 0.3454487996688404, 0.42828190949553085, 0.9762917691705063, 0.9915532833695049, 0.7049602109909741, 0.6422580812848077, 0.7108281775367439, 0.6054694572923164, 0.6410258155509546, 0.735598423150389, 0.7943034312635242, 0.5064252356126562, 0.6594792232777923, 0.5849850167511167, 0.5051697437528307, 0.8403938680663743, 1.1855191729159136, 1.3389551941495046, 1.3670211546310675, 1.813701999068781, 2.121586856583169, 2.3029948198315564, 1.7308790820024094, 1.9950656265946884, 1.8078465646053705, 1.1966103442946037, 1.1122912636889644 ], "peak_layer": 27, "peak_normalized": 2.3029948198315564, "population_cv": 0.5622256721194347, "rest_mean": 1.5394255788797402e-05, "spike_contrast": 1.3886079240584825, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.1376585573307237e-05, "top_five_layers": [ 27, 26, 29, 25, 30 ], "top_five_spike_overlap": 1 }, "values": [ 2.6477515348233283e-05, 9.27226574276574e-06, 3.875539277942153e-06, 8.17999170976691e-06, 6.979371391935274e-06, 5.640831659547985e-06, 6.993413080635946e-06, 1.5941863239277154e-05, 1.6191068425541744e-05, 1.1511291631904896e-05, 1.0487428880878724e-05, 1.160710962722078e-05, 9.886707630357705e-06, 1.04673072200967e-05, 1.2011582839477342e-05, 1.2970176612725481e-05, 8.269415047834627e-06, 1.076863281923579e-05, 9.552217306918465e-06, 8.248914127761964e-06, 1.3722787116421387e-05, 1.9358336430741474e-05, 2.1863792426302098e-05, 2.2322081349557266e-05, 2.961593054351397e-05, 3.4643380786292255e-05, 3.7605590478051454e-05, 2.826351555995643e-05, 3.2577416277490556e-05, 2.9520317184505984e-05, 1.9539444110705517e-05, 1.8162598280468956e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 5.0574135912029305e-05, "first_to_last_ratio": 0.2537347538630065, "imbalance_abs_log_ratio": 1.3714658337297103, "last_quartile_mean": 0.00019931891529267887, "mean": 9.536071013371838e-05, "normalized": [ 1.3126450055964778, 0.4952527628291807, 0.2625321408364328, 0.5033312969878576, 0.31530609350462707, 0.2632938942014061, 0.3566396684731473, 0.7337641791576914, 0.8008354041476243, 0.5992656402378652, 0.5473281951962525, 0.5568228447878986, 0.48408864590288603, 0.5331099308233739, 0.6305800268446531, 0.6428377419660309, 0.43926431968733415, 0.5642341032181795, 0.5378476228296811, 0.4006437006682332, 0.7479025199898355, 1.0647824587026904, 1.2015174070759518, 1.284913406294966, 1.9132137619701015, 2.6076493025418688, 3.26889372206155, 2.3005738268212834, 2.1251072057644445, 1.9446206996555635, 1.3259632050068015, 1.2352392662181098 ], "peak_layer": 27, "peak_normalized": 3.26889372206155, "population_cv": 0.7465220370071499, "rest_mean": 9.107890986746902e-05, "spike_contrast": 1.3008766985010192, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00011848243157146499, "top_five_layers": [ 27, 26, 28, 29, 30 ], "top_five_spike_overlap": 0 }, "values": [ 0.00012517475988715887, 4.7227655159076676e-05, 2.5035251383087598e-05, 4.799802991328761e-05, 3.0067812986089848e-05, 2.51078927249182e-05, 3.4009412047453225e-05, 6.997227319516242e-05, 7.636823283974081e-05, 5.714639701182023e-05, 5.219360537012108e-05, 5.3099021897651255e-05, 4.616303704096936e-05, 5.0837741582654417e-05, 6.013255915604532e-05, 6.130146357463673e-05, 4.188855746178888e-05, 5.380576476454735e-05, 5.1289531256770715e-05, 3.820566780632362e-05, 7.132051541702822e-05, 0.00010153841139981523, 0.00011457755317678675, 0.000122530254884623, 0.00018244542297907174, 0.0002486672892700881, 0.00031172402668744326, 0.00021938435384072363, 0.0002026517322519794, 0.00018544041085988283, 0.00012644479284062982, 0.00011779329361161217 ] } } } }, "selector": { "expected_uniform_count": 64, "identities_unique": true, "identity_order_sha256": "97fd9721a263024f6a746638f32e0ad9a988cd4c111c00e9e178c1022e5c9d5e", "passed": true, "uniform_count": 64, "uniform_indices": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63 ], "visit_count": 65, "visits": [ { "branch": "attention", "coefficient": "uniform", "group": 1, "index": 0, "kind": "depth", "layer": 1, "offset": 1, "sources": 1, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 1, "index": 1, "kind": "depth", "layer": 1, "offset": 1, "sources": 2, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 1, "index": 2, "kind": "depth", "layer": 2, "offset": 2, "sources": 2, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 1, "index": 3, "kind": "depth", "layer": 2, "offset": 2, "sources": 2, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 1, "index": 4, "kind": "depth", "layer": 3, "offset": 3, "sources": 2, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 1, "index": 5, "kind": "depth", "layer": 3, "offset": 3, "sources": 2, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 1, "index": 6, "kind": "depth", "layer": 4, "offset": 4, "sources": 2, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 1, "index": 7, "kind": "depth", "layer": 4, "offset": 4, "sources": 2, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 2, "index": 8, "kind": "depth", "layer": 5, "offset": 1, "sources": 2, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 2, "index": 9, "kind": "depth", "layer": 5, "offset": 1, "sources": 3, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 2, "index": 10, "kind": "depth", "layer": 6, "offset": 2, "sources": 3, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 2, "index": 11, "kind": "depth", "layer": 6, "offset": 2, "sources": 3, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 2, "index": 12, "kind": "depth", "layer": 7, "offset": 3, "sources": 3, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 2, "index": 13, "kind": "depth", "layer": 7, "offset": 3, "sources": 3, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 2, "index": 14, "kind": "depth", "layer": 8, "offset": 4, "sources": 3, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 2, "index": 15, "kind": "depth", "layer": 8, "offset": 4, "sources": 3, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 3, "index": 16, "kind": "depth", "layer": 9, "offset": 1, "sources": 3, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 3, "index": 17, "kind": "depth", "layer": 9, "offset": 1, "sources": 4, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 3, "index": 18, "kind": "depth", "layer": 10, "offset": 2, "sources": 4, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 3, "index": 19, "kind": "depth", "layer": 10, "offset": 2, "sources": 4, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 3, "index": 20, "kind": "depth", "layer": 11, "offset": 3, "sources": 4, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 3, "index": 21, "kind": "depth", "layer": 11, "offset": 3, "sources": 4, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 3, "index": 22, "kind": "depth", "layer": 12, "offset": 4, "sources": 4, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 3, "index": 23, "kind": "depth", "layer": 12, "offset": 4, "sources": 4, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 4, "index": 24, "kind": "depth", "layer": 13, "offset": 1, "sources": 4, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 4, "index": 25, "kind": "depth", "layer": 13, "offset": 1, "sources": 5, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 4, "index": 26, "kind": "depth", "layer": 14, "offset": 2, "sources": 5, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 4, "index": 27, "kind": "depth", "layer": 14, "offset": 2, "sources": 5, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 4, "index": 28, "kind": "depth", "layer": 15, "offset": 3, "sources": 5, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 4, "index": 29, "kind": "depth", "layer": 15, "offset": 3, "sources": 5, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 4, "index": 30, "kind": "depth", "layer": 16, "offset": 4, "sources": 5, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 4, "index": 31, "kind": "depth", "layer": 16, "offset": 4, "sources": 5, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 5, "index": 32, "kind": "depth", "layer": 17, "offset": 1, "sources": 5, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 5, "index": 33, "kind": "depth", "layer": 17, "offset": 1, "sources": 6, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 5, "index": 34, "kind": "depth", "layer": 18, "offset": 2, "sources": 6, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 5, "index": 35, "kind": "depth", "layer": 18, "offset": 2, "sources": 6, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 5, "index": 36, "kind": "depth", "layer": 19, "offset": 3, "sources": 6, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 5, "index": 37, "kind": "depth", "layer": 19, "offset": 3, "sources": 6, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 5, "index": 38, "kind": "depth", "layer": 20, "offset": 4, "sources": 6, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 5, "index": 39, "kind": "depth", "layer": 20, "offset": 4, "sources": 6, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 6, "index": 40, "kind": "depth", "layer": 21, "offset": 1, "sources": 6, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 6, "index": 41, "kind": "depth", "layer": 21, "offset": 1, "sources": 7, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 6, "index": 42, "kind": "depth", "layer": 22, "offset": 2, "sources": 7, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 6, "index": 43, "kind": "depth", "layer": 22, "offset": 2, "sources": 7, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 6, "index": 44, "kind": "depth", "layer": 23, "offset": 3, "sources": 7, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 6, "index": 45, "kind": "depth", "layer": 23, "offset": 3, "sources": 7, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 6, "index": 46, "kind": "depth", "layer": 24, "offset": 4, "sources": 7, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 6, "index": 47, "kind": "depth", "layer": 24, "offset": 4, "sources": 7, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 7, "index": 48, "kind": "depth", "layer": 25, "offset": 1, "sources": 7, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 7, "index": 49, "kind": "depth", "layer": 25, "offset": 1, "sources": 8, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 7, "index": 50, "kind": "depth", "layer": 26, "offset": 2, "sources": 8, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 7, "index": 51, "kind": "depth", "layer": 26, "offset": 2, "sources": 8, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 7, "index": 52, "kind": "depth", "layer": 27, "offset": 3, "sources": 8, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 7, "index": 53, "kind": "depth", "layer": 27, "offset": 3, "sources": 8, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 7, "index": 54, "kind": "depth", "layer": 28, "offset": 4, "sources": 8, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 7, "index": 55, "kind": "depth", "layer": 28, "offset": 4, "sources": 8, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 8, "index": 56, "kind": "depth", "layer": 29, "offset": 1, "sources": 8, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 8, "index": 57, "kind": "depth", "layer": 29, "offset": 1, "sources": 9, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 8, "index": 58, "kind": "depth", "layer": 30, "offset": 2, "sources": 9, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 8, "index": 59, "kind": "depth", "layer": 30, "offset": 2, "sources": 9, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 8, "index": 60, "kind": "depth", "layer": 31, "offset": 3, "sources": 9, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 8, "index": 61, "kind": "depth", "layer": 31, "offset": 3, "sources": 9, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 8, "index": 62, "kind": "depth", "layer": 32, "offset": 4, "sources": 9, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 8, "index": 63, "kind": "depth", "layer": 32, "offset": 4, "sources": 9, "use_uniform": true }, { "branch": "output", "coefficient": "detached_learned", "group": null, "index": 64, "kind": "output", "layer": null, "offset": null, "sources": 9, "use_uniform": false } ], "visits_sha256": "07670769be21c5fc9c1f1c3f868748e2c48b84de5b27352f602103abf0cbce67" } }, "uniform_except_group_6": { "bits_per_byte": 1.8740129107275643, "forward": { "activation_sha256": { "attention_branch_output": "5e6ecbcd78543fc42d4098a51169dca446d404ce384015643e84d2c78433c949", "mlp_branch_output": "c0b8018c349b1bd09e5d17858eb312a2323da0e92de9651e2bb08ae4adf115c7", "post_attention_state": "90a633f12d984935028fe5d7212be48e7da7b4a0806751939a551938d16dc540", "post_mlp_state": "89f2226456bfe96e245d2aad0f699d32c676f9f53099789cd870253d002402f5", "pre_attention_input": "ee695e78937de01b7128d3866b8fc5dd328ebdf2c159be37918d4f02952d5248", "pre_mlp_input": "e53fa425d89dfeb0e9395eca9bcd73dc0a55314abf45c5b10b43e2cbfa724345" }, "logits_sha256": "bdacba5675f0579cec629bbf01b36c8660ad576cfdf9fef1bea8259eaeac9ec2", "loss_nats": 1.2989667654037476, "mixer_summary_sha256": "eb3ac0f07de9b755d6e67a510ab68d09d9ad666bb7b5d020ba7d8c5055c399b2" }, "loss_nats": 1.2989667654037476, "loss_scale": 1.0, "mixers": [ { "branch": "attention", "entropy_mean": -0.0, "group": 1, "latest_source_mass": 1.0, "layer": 1, "max_source_mass": 1.0, "mixer_index": 0, "normalized_entropy": 1.0, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 1.0, "median": 1.0, "p05": 1.0, "p95": 1.0 } ], "sources": 1 }, { "branch": "mlp", "entropy_mean": 0.22886337339878082, "group": 1, "latest_source_mass": 0.06350618600845337, "layer": 1, "max_source_mass": 0.9364938139915466, "mixer_index": 1, "normalized_entropy": 0.3301800538435402, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.9364938139915466, "median": 0.9452826976776123, "p05": 0.8725647330284119, "p95": 0.9712384939193726 }, { "label": "current_group_partial", "mean": 0.06350618600845337, "median": 0.0547172985970974, "p05": 0.028761589899659157, "p95": 0.12743529677391052 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.603729784488678, "group": 1, "latest_source_mass": 0.4133155345916748, "layer": 2, "max_source_mass": 0.5866844654083252, "mixer_index": 2, "normalized_entropy": 0.8709979661187783, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.5866844654083252, "median": 0.5734395384788513, "p05": 0.29813364148139954, "p95": 0.8843642473220825 }, { "label": "current_group_partial", "mean": 0.4133155345916748, "median": 0.4265604615211487, "p05": 0.11563576757907867, "p95": 0.7018663287162781 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.3967982828617096, "group": 1, "latest_source_mass": 0.14110636711120605, "layer": 2, "max_source_mass": 0.858893632888794, "mixer_index": 3, "normalized_entropy": 0.5724589149178446, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.858893632888794, "median": 0.8705973029136658, "p05": 0.7599387168884277, "p95": 0.9270997643470764 }, { "label": "current_group_partial", "mean": 0.14110636711120605, "median": 0.12940272688865662, "p05": 0.07290027290582657, "p95": 0.2400612235069275 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6852496862411499, "group": 1, "latest_source_mass": 0.518975019454956, "layer": 3, "max_source_mass": 0.518975019454956, "mixer_index": 4, "normalized_entropy": 0.9886063241108252, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.48102498054504395, "median": 0.48102718591690063, "p05": 0.38786420226097107, "p95": 0.580143928527832 }, { "label": "current_group_partial", "mean": 0.518975019454956, "median": 0.5189728140830994, "p05": 0.41985607147216797, "p95": 0.6121357679367065 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.5890734195709229, "group": 1, "latest_source_mass": 0.3123423159122467, "layer": 3, "max_source_mass": 0.6876576542854309, "mixer_index": 5, "normalized_entropy": 0.8498533011344741, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.6876576542854309, "median": 0.6909762620925903, "p05": 0.48803940415382385, "p95": 0.8900076746940613 }, { "label": "current_group_partial", "mean": 0.3123423159122467, "median": 0.3090237081050873, "p05": 0.10999235510826111, "p95": 0.5119606256484985 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6873323321342468, "group": 1, "latest_source_mass": 0.4831485152244568, "layer": 4, "max_source_mass": 0.5168514251708984, "mixer_index": 6, "normalized_entropy": 0.9916109470127238, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.5168514251708984, "median": 0.5219584703445435, "p05": 0.4251725673675537, "p95": 0.5947866439819336 }, { "label": "current_group_partial", "mean": 0.4831485152244568, "median": 0.47804147005081177, "p05": 0.405213326215744, "p95": 0.5748274326324463 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.487301766872406, "group": 1, "latest_source_mass": 0.2060384750366211, "layer": 4, "max_source_mass": 0.7939615249633789, "mixer_index": 7, "normalized_entropy": 0.7030278424832499, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.7939615249633789, "median": 0.7953404188156128, "p05": 0.6590759754180908, "p95": 0.9329437017440796 }, { "label": "current_group_partial", "mean": 0.2060384750366211, "median": 0.20465964078903198, "p05": 0.06705628335475922, "p95": 0.3409239649772644 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.674697756767273, "group": 2, "latest_source_mass": 0.42952626943588257, "layer": 5, "max_source_mass": 0.5704737305641174, "mixer_index": 8, "normalized_entropy": 0.9733831077870527, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.5704737305641174, "median": 0.5696132779121399, "p05": 0.4695202708244324, "p95": 0.6750621199607849 }, { "label": "completed_group_1", "mean": 0.42952626943588257, "median": 0.4303867816925049, "p05": 0.3249378800392151, "p95": 0.5304797887802124 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.9691112041473389, "group": 2, "latest_source_mass": 0.2681158185005188, "layer": 5, "max_source_mass": 0.5506307482719421, "mixer_index": 9, "normalized_entropy": 0.8821230329784768, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.5506307482719421, "median": 0.543993353843689, "p05": 0.41262108087539673, "p95": 0.7161091566085815 }, { "label": "completed_group_1", "mean": 0.18125343322753906, "median": 0.1794438362121582, "p05": 0.11575750261545181, "p95": 0.25031501054763794 }, { "label": "current_group_partial", "mean": 0.2681158185005188, "median": 0.2664111256599426, "p05": 0.1564716249704361, "p95": 0.3817881643772125 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0745112895965576, "group": 2, "latest_source_mass": 0.412794291973114, "layer": 6, "max_source_mass": 0.412794291973114, "mixer_index": 10, "normalized_entropy": 0.9780623252441762, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.34262996912002563, "median": 0.3412924110889435, "p05": 0.29556700587272644, "p95": 0.39125555753707886 }, { "label": "completed_group_1", "mean": 0.24457570910453796, "median": 0.24382859468460083, "p05": 0.22120657563209534, "p95": 0.2734297513961792 }, { "label": "current_group_partial", "mean": 0.412794291973114, "median": 0.4151499271392822, "p05": 0.37735897302627563, "p95": 0.43749260902404785 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0074045658111572, "group": 2, "latest_source_mass": 0.3775783181190491, "layer": 6, "max_source_mass": 0.4399106502532959, "mixer_index": 11, "normalized_entropy": 0.9169791528842925, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.4399106502532959, "median": 0.44856905937194824, "p05": 0.22063353657722473, "p95": 0.6359403133392334 }, { "label": "completed_group_1", "mean": 0.18251106142997742, "median": 0.18562838435173035, "p05": 0.1196163073182106, "p95": 0.23045453429222107 }, { "label": "current_group_partial", "mean": 0.3775783181190491, "median": 0.3675365447998047, "p05": 0.23817461729049683, "p95": 0.5557451844215393 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.060497522354126, "group": 2, "latest_source_mass": 0.4025024175643921, "layer": 7, "max_source_mass": 0.4025024175643921, "mixer_index": 12, "normalized_entropy": 0.9653064445872968, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.3786153197288513, "median": 0.3907395303249359, "p05": 0.28039106726646423, "p95": 0.4436405599117279 }, { "label": "completed_group_1", "mean": 0.2188822478055954, "median": 0.21226292848587036, "p05": 0.1885872334241867, "p95": 0.2672030031681061 }, { "label": "current_group_partial", "mean": 0.4025024175643921, "median": 0.39815032482147217, "p05": 0.3533596992492676, "p95": 0.460100919008255 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 0.9941019415855408, "group": 2, "latest_source_mass": 0.4801652431488037, "layer": 7, "max_source_mass": 0.4801652431488037, "mixer_index": 13, "normalized_entropy": 0.9048705824970601, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.32296955585479736, "median": 0.2803739905357361, "p05": 0.1784922033548355, "p95": 0.6440245509147644 }, { "label": "completed_group_1", "mean": 0.19686520099639893, "median": 0.2053825855255127, "p05": 0.09813719242811203, "p95": 0.2504696547985077 }, { "label": "current_group_partial", "mean": 0.4801652431488037, "median": 0.5057140588760376, "p05": 0.2529003322124481, "p95": 0.598870575428009 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0503522157669067, "group": 2, "latest_source_mass": 0.4001518487930298, "layer": 8, "max_source_mass": 0.40130966901779175, "mixer_index": 14, "normalized_entropy": 0.9560717885654542, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.40130966901779175, "median": 0.396759569644928, "p05": 0.3479611873626709, "p95": 0.4733521044254303 }, { "label": "completed_group_1", "mean": 0.19853851199150085, "median": 0.19769755005836487, "p05": 0.17136667668819427, "p95": 0.22788912057876587 }, { "label": "current_group_partial", "mean": 0.4001518487930298, "median": 0.40291696786880493, "p05": 0.3506356179714203, "p95": 0.43941178917884827 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0129402875900269, "group": 2, "latest_source_mass": 0.42161279916763306, "layer": 8, "max_source_mass": 0.42161279916763306, "mixer_index": 15, "normalized_entropy": 0.9220179839951123, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.4093267023563385, "median": 0.4046899676322937, "p05": 0.2895161807537079, "p95": 0.5762131810188293 }, { "label": "completed_group_1", "mean": 0.16906049847602844, "median": 0.16888955235481262, "p05": 0.11233551055192947, "p95": 0.22477403283119202 }, { "label": "current_group_partial", "mean": 0.42161279916763306, "median": 0.4227708578109741, "p05": 0.3131049573421478, "p95": 0.519075334072113 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0774019956588745, "group": 3, "latest_source_mass": 0.3560434877872467, "layer": 9, "max_source_mass": 0.38510820269584656, "mixer_index": 16, "normalized_entropy": 0.9806935592947451, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.38510820269584656, "median": 0.3892287313938141, "p05": 0.2934730052947998, "p95": 0.4718902111053467 }, { "label": "completed_group_1", "mean": 0.25884830951690674, "median": 0.25734007358551025, "p05": 0.22217319905757904, "p95": 0.29551801085472107 }, { "label": "completed_group_2", "mean": 0.3560434877872467, "median": 0.35412120819091797, "p05": 0.29444968700408936, "p95": 0.41908568143844604 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.2740377187728882, "group": 3, "latest_source_mass": 0.3633861243724823, "layer": 9, "max_source_mass": 0.3633861243724823, "mixer_index": 17, "normalized_entropy": 0.9190239493895668, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.29550763964653015, "median": 0.30360478162765503, "p05": 0.16904757916927338, "p95": 0.45146965980529785 }, { "label": "completed_group_1", "mean": 0.09541244804859161, "median": 0.09486871212720871, "p05": 0.061360105872154236, "p95": 0.13037073612213135 }, { "label": "completed_group_2", "mean": 0.24569380283355713, "median": 0.24333015084266663, "p05": 0.1698218435049057, "p95": 0.31589236855506897 }, { "label": "current_group_partial", "mean": 0.3633861243724823, "median": 0.36472851037979126, "p05": 0.2795379161834717, "p95": 0.4399876892566681 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.360302448272705, "group": 3, "latest_source_mass": 0.28569501638412476, "layer": 10, "max_source_mass": 0.3109527826309204, "mixer_index": 18, "normalized_entropy": 0.9812507981160736, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.3109527826309204, "median": 0.32172977924346924, "p05": 0.23520725965499878, "p95": 0.37052470445632935 }, { "label": "completed_group_1", "mean": 0.20984819531440735, "median": 0.20958667993545532, "p05": 0.1891367882490158, "p95": 0.23184512555599213 }, { "label": "completed_group_2", "mean": 0.19350400567054749, "median": 0.19153150916099548, "p05": 0.16607457399368286, "p95": 0.22877082228660583 }, { "label": "current_group_partial", "mean": 0.28569501638412476, "median": 0.28060320019721985, "p05": 0.23824527859687805, "p95": 0.3439165949821472 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.2722704410552979, "group": 3, "latest_source_mass": 0.29912811517715454, "layer": 10, "max_source_mass": 0.35036465525627136, "mixer_index": 19, "normalized_entropy": 0.9177491279900463, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.35036465525627136, "median": 0.36210548877716064, "p05": 0.12807878851890564, "p95": 0.5797606706619263 }, { "label": "completed_group_1", "mean": 0.12631990015506744, "median": 0.1264108121395111, "p05": 0.06977735459804535, "p95": 0.1755298376083374 }, { "label": "completed_group_2", "mean": 0.22418735921382904, "median": 0.22563719749450684, "p05": 0.1176326721906662, "p95": 0.32001155614852905 }, { "label": "current_group_partial", "mean": 0.29912811517715454, "median": 0.28589946031570435, "p05": 0.193109393119812, "p95": 0.44119665026664734 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3558036088943481, "group": 3, "latest_source_mass": 0.314197838306427, "layer": 11, "max_source_mass": 0.314197838306427, "mixer_index": 20, "normalized_entropy": 0.978005571485618, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.298976868391037, "median": 0.2961939573287964, "p05": 0.24950163066387177, "p95": 0.35835936665534973 }, { "label": "completed_group_1", "mean": 0.1941717267036438, "median": 0.1918032318353653, "p05": 0.16809208691120148, "p95": 0.22895419597625732 }, { "label": "completed_group_2", "mean": 0.1926535815000534, "median": 0.19306811690330505, "p05": 0.16983944177627563, "p95": 0.2155935913324356 }, { "label": "current_group_partial", "mean": 0.314197838306427, "median": 0.31541913747787476, "p05": 0.2673022747039795, "p95": 0.35941022634506226 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.2696614265441895, "group": 3, "latest_source_mass": 0.2862822115421295, "layer": 11, "max_source_mass": 0.3667730689048767, "mixer_index": 21, "normalized_entropy": 0.9158671218416545, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.3667730689048767, "median": 0.4067659378051758, "p05": 0.10593366622924805, "p95": 0.5672274827957153 }, { "label": "completed_group_1", "mean": 0.15310388803482056, "median": 0.15050218999385834, "p05": 0.08651046454906464, "p95": 0.22459912300109863 }, { "label": "completed_group_2", "mean": 0.19384080171585083, "median": 0.18732509016990662, "p05": 0.1267009675502777, "p95": 0.281107634305954 }, { "label": "current_group_partial", "mean": 0.2862822115421295, "median": 0.26271265745162964, "p05": 0.17024393379688263, "p95": 0.475044310092926 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3606936931610107, "group": 3, "latest_source_mass": 0.3028184771537781, "layer": 12, "max_source_mass": 0.3028184771537781, "mixer_index": 22, "normalized_entropy": 0.9815330216461395, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.298227459192276, "median": 0.2954302132129669, "p05": 0.25287362933158875, "p95": 0.35405808687210083 }, { "label": "completed_group_1", "mean": 0.18611131608486176, "median": 0.1850002110004425, "p05": 0.15901757776737213, "p95": 0.21725836396217346 }, { "label": "completed_group_2", "mean": 0.21284276247024536, "median": 0.21402311325073242, "p05": 0.18819917738437653, "p95": 0.23418807983398438 }, { "label": "current_group_partial", "mean": 0.3028184771537781, "median": 0.3041122555732727, "p05": 0.25661808252334595, "p95": 0.34458184242248535 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.269316554069519, "group": 3, "latest_source_mass": 0.2653314769268036, "layer": 12, "max_source_mass": 0.4081852436065674, "mixer_index": 23, "normalized_entropy": 0.9156183489371815, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.4081852436065674, "median": 0.4254342317581177, "p05": 0.20765496790409088, "p95": 0.587238073348999 }, { "label": "completed_group_1", "mean": 0.14420127868652344, "median": 0.1438048928976059, "p05": 0.1064920499920845, "p95": 0.1871720254421234 }, { "label": "completed_group_2", "mean": 0.18228203058242798, "median": 0.17507661879062653, "p05": 0.12183455377817154, "p95": 0.26505547761917114 }, { "label": "current_group_partial", "mean": 0.2653314769268036, "median": 0.24924281239509583, "p05": 0.15609969198703766, "p95": 0.4061311185359955 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3636136054992676, "group": 4, "latest_source_mass": 0.3136237859725952, "layer": 13, "max_source_mass": 0.3136237859725952, "mixer_index": 24, "normalized_entropy": 0.9836392931712563, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.27427661418914795, "median": 0.27403637766838074, "p05": 0.22928935289382935, "p95": 0.319773405790329 }, { "label": "completed_group_1", "mean": 0.21341991424560547, "median": 0.20697325468063354, "p05": 0.18552586436271667, "p95": 0.26688331365585327 }, { "label": "completed_group_2", "mean": 0.19867971539497375, "median": 0.197545126080513, "p05": 0.1794913411140442, "p95": 0.222100168466568 }, { "label": "completed_group_3", "mean": 0.3136237859725952, "median": 0.31875866651535034, "p05": 0.24156072735786438, "p95": 0.3695686459541321 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.570773959159851, "group": 4, "latest_source_mass": 0.2294243425130844, "layer": 13, "max_source_mass": 0.2658740282058716, "mixer_index": 25, "normalized_entropy": 0.9759767351225285, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.2658740282058716, "median": 0.2747163474559784, "p05": 0.17927880585193634, "p95": 0.3512483835220337 }, { "label": "completed_group_1", "mean": 0.13419361412525177, "median": 0.13374894857406616, "p05": 0.11072897166013718, "p95": 0.1586270034313202 }, { "label": "completed_group_2", "mean": 0.1907704919576645, "median": 0.19074805080890656, "p05": 0.14699944853782654, "p95": 0.23316305875778198 }, { "label": "completed_group_3", "mean": 0.17973749339580536, "median": 0.17616114020347595, "p05": 0.1363743096590042, "p95": 0.23202598094940186 }, { "label": "current_group_partial", "mean": 0.2294243425130844, "median": 0.22995705902576447, "p05": 0.17975632846355438, "p95": 0.2789157032966614 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.5560038089752197, "group": 4, "latest_source_mass": 0.32468360662460327, "layer": 14, "max_source_mass": 0.32468360662460327, "mixer_index": 26, "normalized_entropy": 0.9667995248241249, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.20477107167243958, "median": 0.206403911113739, "p05": 0.16619175672531128, "p95": 0.2431035190820694 }, { "label": "completed_group_1", "mean": 0.15721669793128967, "median": 0.15658551454544067, "p05": 0.13878680765628815, "p95": 0.17778633534908295 }, { "label": "completed_group_2", "mean": 0.13811248540878296, "median": 0.13743364810943604, "p05": 0.12352626770734787, "p95": 0.15553249418735504 }, { "label": "completed_group_3", "mean": 0.1752161681652069, "median": 0.17309701442718506, "p05": 0.15223677456378937, "p95": 0.20474471151828766 }, { "label": "current_group_partial", "mean": 0.32468360662460327, "median": 0.32497870922088623, "p05": 0.2946181893348694, "p95": 0.35203880071640015 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.5659973621368408, "group": 4, "latest_source_mass": 0.26703086495399475, "layer": 14, "max_source_mass": 0.26703086495399475, "mixer_index": 27, "normalized_entropy": 0.9730088685238187, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.20531430840492249, "median": 0.20229148864746094, "p05": 0.10698921978473663, "p95": 0.3393132984638214 }, { "label": "completed_group_1", "mean": 0.15986156463623047, "median": 0.15952220559120178, "p05": 0.11969029903411865, "p95": 0.20180393755435944 }, { "label": "completed_group_2", "mean": 0.21557100117206573, "median": 0.21654027700424194, "p05": 0.1673085242509842, "p95": 0.2610440254211426 }, { "label": "completed_group_3", "mean": 0.15222224593162537, "median": 0.1510905921459198, "p05": 0.11800260841846466, "p95": 0.18893754482269287 }, { "label": "current_group_partial", "mean": 0.26703086495399475, "median": 0.258539617061615, "p05": 0.19330690801143646, "p95": 0.36521556973457336 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.5552496910095215, "group": 4, "latest_source_mass": 0.31774693727493286, "layer": 15, "max_source_mass": 0.31774693727493286, "mixer_index": 28, "normalized_entropy": 0.9663309649872576, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.2162160575389862, "median": 0.2145705670118332, "p05": 0.17727568745613098, "p95": 0.26878347992897034 }, { "label": "completed_group_1", "mean": 0.17982497811317444, "median": 0.18024154007434845, "p05": 0.15453973412513733, "p95": 0.2030555009841919 }, { "label": "completed_group_2", "mean": 0.1307673156261444, "median": 0.13107559084892273, "p05": 0.11463287472724915, "p95": 0.14630235731601715 }, { "label": "completed_group_3", "mean": 0.1554446816444397, "median": 0.15516987442970276, "p05": 0.13627474009990692, "p95": 0.1754152774810791 }, { "label": "current_group_partial", "mean": 0.31774693727493286, "median": 0.319039523601532, "p05": 0.27606821060180664, "p95": 0.35650834441185 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.5068919658660889, "group": 4, "latest_source_mass": 0.3298327624797821, "layer": 15, "max_source_mass": 0.3298327624797821, "mixer_index": 29, "normalized_entropy": 0.9362846209998111, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.20646697282791138, "median": 0.18775638937950134, "p05": 0.0806753933429718, "p95": 0.4274584949016571 }, { "label": "completed_group_1", "mean": 0.1197461187839508, "median": 0.11989086866378784, "p05": 0.0809398666024208, "p95": 0.15743419528007507 }, { "label": "completed_group_2", "mean": 0.19215428829193115, "median": 0.19620640575885773, "p05": 0.1148109883069992, "p95": 0.2491212636232376 }, { "label": "completed_group_3", "mean": 0.15179988741874695, "median": 0.15212899446487427, "p05": 0.10697861760854721, "p95": 0.19542087614536285 }, { "label": "current_group_partial", "mean": 0.3298327624797821, "median": 0.32837340235710144, "p05": 0.22746729850769043, "p95": 0.4339301884174347 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.5435223579406738, "group": 4, "latest_source_mass": 0.3224271535873413, "layer": 16, "max_source_mass": 0.3224271535873413, "mixer_index": 30, "normalized_entropy": 0.9590443632623663, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.22410975396633148, "median": 0.22760415077209473, "p05": 0.14593973755836487, "p95": 0.30093055963516235 }, { "label": "completed_group_1", "mean": 0.16986176371574402, "median": 0.16963240504264832, "p05": 0.13569067418575287, "p95": 0.20651353895664215 }, { "label": "completed_group_2", "mean": 0.13080403208732605, "median": 0.13075244426727295, "p05": 0.10945598781108856, "p95": 0.15187400579452515 }, { "label": "completed_group_3", "mean": 0.15279728174209595, "median": 0.15036612749099731, "p05": 0.12810495495796204, "p95": 0.18468116223812103 }, { "label": "current_group_partial", "mean": 0.3224271535873413, "median": 0.3164973258972168, "p05": 0.2600978910923004, "p95": 0.40530598163604736 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.4949920177459717, "group": 4, "latest_source_mass": 0.31847020983695984, "layer": 16, "max_source_mass": 0.31847020983695984, "mixer_index": 31, "normalized_entropy": 0.9288907675133354, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.27174079418182373, "median": 0.2585451304912567, "p05": 0.13639721274375916, "p95": 0.4698348045349121 }, { "label": "completed_group_1", "mean": 0.12178683280944824, "median": 0.11714950203895569, "p05": 0.08594125509262085, "p95": 0.1723237782716751 }, { "label": "completed_group_2", "mean": 0.1529826819896698, "median": 0.15223106741905212, "p05": 0.10371473431587219, "p95": 0.203575000166893 }, { "label": "completed_group_3", "mean": 0.1350194811820984, "median": 0.13363957405090332, "p05": 0.08964549005031586, "p95": 0.18468207120895386 }, { "label": "current_group_partial", "mean": 0.31847020983695984, "median": 0.3185216188430786, "p05": 0.21444711089134216, "p95": 0.4301546812057495 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.544264793395996, "group": 5, "latest_source_mass": 0.2861037850379944, "layer": 17, "max_source_mass": 0.2861037850379944, "mixer_index": 32, "normalized_entropy": 0.9595056643474137, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.24016115069389343, "median": 0.23868846893310547, "p05": 0.18435582518577576, "p95": 0.2983453571796417 }, { "label": "completed_group_1", "mean": 0.20730282366275787, "median": 0.1998077780008316, "p05": 0.16255050897598267, "p95": 0.28107208013534546 }, { "label": "completed_group_2", "mean": 0.09692573547363281, "median": 0.0967404842376709, "p05": 0.08098706603050232, "p95": 0.11221462488174438 }, { "label": "completed_group_3", "mean": 0.1695065200328827, "median": 0.1696585714817047, "p05": 0.136396586894989, "p95": 0.20130634307861328 }, { "label": "completed_group_4", "mean": 0.2861037850379944, "median": 0.28883200883865356, "p05": 0.22401469945907593, "p95": 0.3409370183944702 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.686682939529419, "group": 5, "latest_source_mass": 0.12281867861747742, "layer": 17, "max_source_mass": 0.2885604202747345, "mixer_index": 33, "normalized_entropy": 0.9413556721740635, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.16755425930023193, "median": 0.1327776312828064, "p05": 0.0790192037820816, "p95": 0.3667412996292114 }, { "label": "completed_group_1", "mean": 0.10962115228176117, "median": 0.10525335371494293, "p05": 0.07840172946453094, "p95": 0.15405339002609253 }, { "label": "completed_group_2", "mean": 0.1783454418182373, "median": 0.17896884679794312, "p05": 0.10802385956048965, "p95": 0.24048911035060883 }, { "label": "completed_group_3", "mean": 0.13310004770755768, "median": 0.13033035397529602, "p05": 0.0904831737279892, "p95": 0.17934636771678925 }, { "label": "completed_group_4", "mean": 0.2885604202747345, "median": 0.2831825315952301, "p05": 0.16967125236988068, "p95": 0.4189499616622925 }, { "label": "current_group_partial", "mean": 0.12281867861747742, "median": 0.12062528729438782, "p05": 0.07461707293987274, "p95": 0.1782219111919403 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.7070834636688232, "group": 5, "latest_source_mass": 0.2988377809524536, "layer": 18, "max_source_mass": 0.2988377809524536, "mixer_index": 34, "normalized_entropy": 0.9527414214834803, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.181177020072937, "median": 0.18419909477233887, "p05": 0.13624237477779388, "p95": 0.22192169725894928 }, { "label": "completed_group_1", "mean": 0.14557164907455444, "median": 0.144476979970932, "p05": 0.1293315887451172, "p95": 0.16486899554729462 }, { "label": "completed_group_2", "mean": 0.08318156749010086, "median": 0.0820261612534523, "p05": 0.07088497281074524, "p95": 0.0990293100476265 }, { "label": "completed_group_3", "mean": 0.11424209177494049, "median": 0.11267061531543732, "p05": 0.0979941189289093, "p95": 0.1353992223739624 }, { "label": "completed_group_4", "mean": 0.17698991298675537, "median": 0.17597860097885132, "p05": 0.14837387204170227, "p95": 0.21050144731998444 }, { "label": "current_group_partial", "mean": 0.2988377809524536, "median": 0.2989804744720459, "p05": 0.2721370458602905, "p95": 0.32572072744369507 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.7419235706329346, "group": 5, "latest_source_mass": 0.26016175746917725, "layer": 18, "max_source_mass": 0.26016175746917725, "mixer_index": 35, "normalized_entropy": 0.972186055410333, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1532188355922699, "median": 0.1451343297958374, "p05": 0.11514341086149216, "p95": 0.22103679180145264 }, { "label": "completed_group_1", "mean": 0.11420352011919022, "median": 0.11261284351348877, "p05": 0.0928458571434021, "p95": 0.14123988151550293 }, { "label": "completed_group_2", "mean": 0.16597424447536469, "median": 0.1655328869819641, "p05": 0.13871800899505615, "p95": 0.19611573219299316 }, { "label": "completed_group_3", "mean": 0.1250365972518921, "median": 0.12369821965694427, "p05": 0.10375828295946121, "p95": 0.14912742376327515 }, { "label": "completed_group_4", "mean": 0.18140508234500885, "median": 0.1796584278345108, "p05": 0.14511024951934814, "p95": 0.22286373376846313 }, { "label": "current_group_partial", "mean": 0.26016175746917725, "median": 0.26266711950302124, "p05": 0.1889389455318451, "p95": 0.3234381675720215 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.7127439975738525, "group": 5, "latest_source_mass": 0.2907913327217102, "layer": 19, "max_source_mass": 0.2907913327217102, "mixer_index": 36, "normalized_entropy": 0.9559006256078307, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.18731768429279327, "median": 0.19042012095451355, "p05": 0.13879702985286713, "p95": 0.23792000114917755 }, { "label": "completed_group_1", "mean": 0.16715484857559204, "median": 0.1667211651802063, "p05": 0.14574480056762695, "p95": 0.19026139378547668 }, { "label": "completed_group_2", "mean": 0.08496320247650146, "median": 0.08452782034873962, "p05": 0.07410907745361328, "p95": 0.097386933863163 }, { "label": "completed_group_3", "mean": 0.11362075805664062, "median": 0.11248823255300522, "p05": 0.0977015346288681, "p95": 0.13219426572322845 }, { "label": "completed_group_4", "mean": 0.15615218877792358, "median": 0.15540224313735962, "p05": 0.1289147436618805, "p95": 0.18598109483718872 }, { "label": "current_group_partial", "mean": 0.2907913327217102, "median": 0.2900415062904358, "p05": 0.25822532176971436, "p95": 0.32475030422210693 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.6076834201812744, "group": 5, "latest_source_mass": 0.3738824725151062, "layer": 19, "max_source_mass": 0.3738824725151062, "mixer_index": 37, "normalized_entropy": 0.8972652009334232, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.15032559633255005, "median": 0.12544110417366028, "p05": 0.06976518034934998, "p95": 0.35840943455696106 }, { "label": "completed_group_1", "mean": 0.08624927699565887, "median": 0.08602266013622284, "p05": 0.061998721212148666, "p95": 0.11138717085123062 }, { "label": "completed_group_2", "mean": 0.15736976265907288, "median": 0.15802210569381714, "p05": 0.10121746361255646, "p95": 0.20687000453472137 }, { "label": "completed_group_3", "mean": 0.08584291487932205, "median": 0.0827617198228836, "p05": 0.060561805963516235, "p95": 0.11351694166660309 }, { "label": "completed_group_4", "mean": 0.14632999897003174, "median": 0.1426895558834076, "p05": 0.10106660425662994, "p95": 0.20179018378257751 }, { "label": "current_group_partial", "mean": 0.3738824725151062, "median": 0.3799295425415039, "p05": 0.23631010949611664, "p95": 0.4884527325630188 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.709290862083435, "group": 5, "latest_source_mass": 0.2901708781719208, "layer": 20, "max_source_mass": 0.2901708781719208, "mixer_index": 38, "normalized_entropy": 0.9539733939957076, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.18991519510746002, "median": 0.1924753338098526, "p05": 0.14201581478118896, "p95": 0.23709386587142944 }, { "label": "completed_group_1", "mean": 0.17091524600982666, "median": 0.17006167769432068, "p05": 0.15297070145606995, "p95": 0.19153237342834473 }, { "label": "completed_group_2", "mean": 0.07822999358177185, "median": 0.07778562605381012, "p05": 0.06707640737295151, "p95": 0.09089364111423492 }, { "label": "completed_group_3", "mean": 0.116765096783638, "median": 0.11516137421131134, "p05": 0.10162825882434845, "p95": 0.13653048872947693 }, { "label": "completed_group_4", "mean": 0.1540035903453827, "median": 0.15287795662879944, "p05": 0.12593276798725128, "p95": 0.18590238690376282 }, { "label": "current_group_partial", "mean": 0.2901708781719208, "median": 0.29104921221733093, "p05": 0.25643545389175415, "p95": 0.3205650746822357 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.6690887212753296, "group": 5, "latest_source_mass": 0.3342471718788147, "layer": 20, "max_source_mass": 0.3342471718788147, "mixer_index": 39, "normalized_entropy": 0.9315361520005944, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.19029226899147034, "median": 0.19644704461097717, "p05": 0.09456793963909149, "p95": 0.30468082427978516 }, { "label": "completed_group_1", "mean": 0.1100241169333458, "median": 0.10808776319026947, "p05": 0.09035106748342514, "p95": 0.1359245330095291 }, { "label": "completed_group_2", "mean": 0.11471512168645859, "median": 0.11494918912649155, "p05": 0.08799737691879272, "p95": 0.14024731516838074 }, { "label": "completed_group_3", "mean": 0.09999909996986389, "median": 0.09682698547840118, "p05": 0.07921088486909866, "p95": 0.1264391839504242 }, { "label": "completed_group_4", "mean": 0.1507222205400467, "median": 0.1476232260465622, "p05": 0.11397356539964676, "p95": 0.19991227984428406 }, { "label": "current_group_partial", "mean": 0.3342471718788147, "median": 0.33802270889282227, "p05": 0.2548266053199768, "p95": 0.40130624175071716 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.7458693981170654, "group": 6, "latest_source_mass": 0.2361394464969635, "layer": 21, "max_source_mass": 0.2361394464969635, "mixer_index": 40, "normalized_entropy": 0.9743882636597644, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.20528709888458252, "median": 0.2140858769416809, "p05": 0.13168928027153015, "p95": 0.27098023891448975 }, { "label": "completed_group_1", "mean": 0.1722964197397232, "median": 0.17238952219486237, "p05": 0.15053534507751465, "p95": 0.19365043938159943 }, { "label": "completed_group_2", "mean": 0.10435444861650467, "median": 0.10361101478338242, "p05": 0.08888213336467743, "p95": 0.12212536484003067 }, { "label": "completed_group_3", "mean": 0.12075909972190857, "median": 0.1180175170302391, "p05": 0.10358668118715286, "p95": 0.14458929002285004 }, { "label": "completed_group_4", "mean": 0.16116347908973694, "median": 0.15841388702392578, "p05": 0.1337081342935562, "p95": 0.19791515171527863 }, { "label": "completed_group_5", "mean": 0.2361394464969635, "median": 0.23709020018577576, "p05": 0.20446214079856873, "p95": 0.26509594917297363 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.4612125158309937, "group": 6, "latest_source_mass": 0.5543455481529236, "layer": 21, "max_source_mass": 0.5543455481529236, "mixer_index": 41, "normalized_entropy": 0.7509146897354808, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.08934466540813446, "median": 0.07911523431539536, "p05": 0.04488275945186615, "p95": 0.1737232506275177 }, { "label": "completed_group_1", "mean": 0.054379627108573914, "median": 0.05344422906637192, "p05": 0.04221750423312187, "p95": 0.06924191862344742 }, { "label": "completed_group_2", "mean": 0.07567165046930313, "median": 0.07438549399375916, "p05": 0.05811355262994766, "p95": 0.09632256627082825 }, { "label": "completed_group_3", "mean": 0.05758609622716904, "median": 0.05612001568078995, "p05": 0.044792283326387405, "p95": 0.07347827404737473 }, { "label": "completed_group_4", "mean": 0.07884950935840607, "median": 0.07757881283760071, "p05": 0.05758974328637123, "p95": 0.10452556610107422 }, { "label": "completed_group_5", "mean": 0.08982288837432861, "median": 0.08912864327430725, "p05": 0.0663854256272316, "p95": 0.11604885756969452 }, { "label": "current_group_partial", "mean": 0.5543455481529236, "median": 0.5596089363098145, "p05": 0.45455610752105713, "p95": 0.6367484331130981 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.7241814136505127, "group": 6, "latest_source_mass": 0.39484071731567383, "layer": 22, "max_source_mass": 0.39484071731567383, "mixer_index": 42, "normalized_entropy": 0.886053970419732, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.13360010087490082, "median": 0.14072942733764648, "p05": 0.07699814438819885, "p95": 0.17506597936153412 }, { "label": "completed_group_1", "mean": 0.10609637200832367, "median": 0.10586297512054443, "p05": 0.08648421615362167, "p95": 0.12602579593658447 }, { "label": "completed_group_2", "mean": 0.06363911926746368, "median": 0.06324908882379532, "p05": 0.054980404675006866, "p95": 0.07316673547029495 }, { "label": "completed_group_3", "mean": 0.07426095008850098, "median": 0.07362888753414154, "p05": 0.06318790465593338, "p95": 0.08665015548467636 }, { "label": "completed_group_4", "mean": 0.09383076429367065, "median": 0.09294568747282028, "p05": 0.07638302445411682, "p95": 0.11445100605487823 }, { "label": "completed_group_5", "mean": 0.13373197615146637, "median": 0.13327741622924805, "p05": 0.11399447172880173, "p95": 0.15641780197620392 }, { "label": "current_group_partial", "mean": 0.39484071731567383, "median": 0.3879151940345764, "p05": 0.3302172124385834, "p95": 0.4852263331413269 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.3979300260543823, "group": 6, "latest_source_mass": 0.5830056071281433, "layer": 22, "max_source_mass": 0.5830056071281433, "mixer_index": 43, "normalized_entropy": 0.7183939231382495, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.08959861099720001, "median": 0.07544538378715515, "p05": 0.04165133833885193, "p95": 0.20099139213562012 }, { "label": "completed_group_1", "mean": 0.05414731800556183, "median": 0.05336771905422211, "p05": 0.04069708660244942, "p95": 0.07046990841627121 }, { "label": "completed_group_2", "mean": 0.07668881118297577, "median": 0.07607124000787735, "p05": 0.0555989071726799, "p95": 0.09980960935354233 }, { "label": "completed_group_3", "mean": 0.05603286623954773, "median": 0.054573021829128265, "p05": 0.04162490367889404, "p95": 0.0730118378996849 }, { "label": "completed_group_4", "mean": 0.06979228556156158, "median": 0.06884932518005371, "p05": 0.04576495662331581, "p95": 0.09806846082210541 }, { "label": "completed_group_5", "mean": 0.07073447108268738, "median": 0.07023533433675766, "p05": 0.04956750199198723, "p95": 0.09379896521568298 }, { "label": "current_group_partial", "mean": 0.5830056071281433, "median": 0.5862616896629333, "p05": 0.4658474922180176, "p95": 0.6849814057350159 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.7459560632705688, "group": 6, "latest_source_mass": 0.37691548466682434, "layer": 23, "max_source_mass": 0.37691548466682434, "mixer_index": 44, "normalized_entropy": 0.897243926765161, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.14318783581256866, "median": 0.1493408977985382, "p05": 0.08292607218027115, "p95": 0.19464610517024994 }, { "label": "completed_group_1", "mean": 0.1094653457403183, "median": 0.10858799517154694, "p05": 0.09432064741849899, "p95": 0.1279931515455246 }, { "label": "completed_group_2", "mean": 0.06349228322505951, "median": 0.06325754523277283, "p05": 0.05439673364162445, "p95": 0.0737837627530098 }, { "label": "completed_group_3", "mean": 0.07859592139720917, "median": 0.0775163471698761, "p05": 0.06603655964136124, "p95": 0.09357087314128876 }, { "label": "completed_group_4", "mean": 0.09567482024431229, "median": 0.09468592703342438, "p05": 0.07671141624450684, "p95": 0.11774884164333344 }, { "label": "completed_group_5", "mean": 0.13266831636428833, "median": 0.13199329376220703, "p05": 0.11244359612464905, "p95": 0.15504513680934906 }, { "label": "current_group_partial", "mean": 0.37691548466682434, "median": 0.37180638313293457, "p05": 0.31207239627838135, "p95": 0.4595233201980591 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.4055393934249878, "group": 6, "latest_source_mass": 0.5772827863693237, "layer": 23, "max_source_mass": 0.5772827863693237, "mixer_index": 45, "normalized_entropy": 0.7223043644164862, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.10154955089092255, "median": 0.08766824007034302, "p05": 0.04901552200317383, "p95": 0.20287738740444183 }, { "label": "completed_group_1", "mean": 0.05619276314973831, "median": 0.055477917194366455, "p05": 0.041608311235904694, "p95": 0.07371112704277039 }, { "label": "completed_group_2", "mean": 0.07791627198457718, "median": 0.07736755907535553, "p05": 0.055920060724020004, "p95": 0.10158214718103409 }, { "label": "completed_group_3", "mean": 0.053915880620479584, "median": 0.05215863138437271, "p05": 0.03798118978738785, "p95": 0.07156119495630264 }, { "label": "completed_group_4", "mean": 0.06398355215787888, "median": 0.06229078769683838, "p05": 0.040616974234580994, "p95": 0.09270931780338287 }, { "label": "completed_group_5", "mean": 0.06915920227766037, "median": 0.0684593990445137, "p05": 0.046986356377601624, "p95": 0.09408729523420334 }, { "label": "current_group_partial", "mean": 0.5772827863693237, "median": 0.5824689269065857, "p05": 0.4518115520477295, "p95": 0.6926202178001404 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.7853620052337646, "group": 6, "latest_source_mass": 0.3407441973686218, "layer": 24, "max_source_mass": 0.3407441973686218, "mixer_index": 46, "normalized_entropy": 0.9174945750195659, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.15284113585948944, "median": 0.1593906283378601, "p05": 0.08678612112998962, "p95": 0.211342453956604 }, { "label": "completed_group_1", "mean": 0.12206703424453735, "median": 0.12153112143278122, "p05": 0.1060115247964859, "p95": 0.1399766057729721 }, { "label": "completed_group_2", "mean": 0.06508444249629974, "median": 0.06482555717229843, "p05": 0.055350616574287415, "p95": 0.07557765394449234 }, { "label": "completed_group_3", "mean": 0.08347313106060028, "median": 0.08190013468265533, "p05": 0.06966532766819, "p95": 0.10050469636917114 }, { "label": "completed_group_4", "mean": 0.09811970591545105, "median": 0.09664031863212585, "p05": 0.07688035070896149, "p95": 0.12369741499423981 }, { "label": "completed_group_5", "mean": 0.13767032325267792, "median": 0.1373325139284134, "p05": 0.11642646044492722, "p95": 0.1606694608926773 }, { "label": "current_group_partial", "mean": 0.3407441973686218, "median": 0.33805108070373535, "p05": 0.271301805973053, "p95": 0.41781002283096313 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.509842872619629, "group": 6, "latest_source_mass": 0.5227814316749573, "layer": 24, "max_source_mass": 0.5227814316749573, "mixer_index": 47, "normalized_entropy": 0.7759057494780099, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.11788781732320786, "median": 0.10097194463014603, "p05": 0.06265684217214584, "p95": 0.2408325970172882 }, { "label": "completed_group_1", "mean": 0.06822571158409119, "median": 0.0673636943101883, "p05": 0.05209728330373764, "p95": 0.08648915588855743 }, { "label": "completed_group_2", "mean": 0.08556114137172699, "median": 0.08463940769433975, "p05": 0.06234566867351532, "p95": 0.1125786080956459 }, { "label": "completed_group_3", "mean": 0.06067090481519699, "median": 0.058974385261535645, "p05": 0.043267179280519485, "p95": 0.07997901737689972 }, { "label": "completed_group_4", "mean": 0.07228855788707733, "median": 0.06972275674343109, "p05": 0.04689513146877289, "p95": 0.106512151658535 }, { "label": "completed_group_5", "mean": 0.07258446514606476, "median": 0.07082539796829224, "p05": 0.050949547439813614, "p95": 0.09970614314079285 }, { "label": "current_group_partial", "mean": 0.5227814316749573, "median": 0.5290273427963257, "p05": 0.38773614168167114, "p95": 0.6392589807510376 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.839367389678955, "group": 7, "latest_source_mass": 0.29583215713500977, "layer": 25, "max_source_mass": 0.29583215713500977, "mixer_index": 48, "normalized_entropy": 0.9452478525649903, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.15906387567520142, "median": 0.15846745669841766, "p05": 0.09519976377487183, "p95": 0.24517810344696045 }, { "label": "completed_group_1", "mean": 0.13139380514621735, "median": 0.1308426856994629, "p05": 0.11478274315595627, "p95": 0.15000829100608826 }, { "label": "completed_group_2", "mean": 0.07717745751142502, "median": 0.07715500891208649, "p05": 0.06616044044494629, "p95": 0.08791667222976685 }, { "label": "completed_group_3", "mean": 0.09607303142547607, "median": 0.09436552226543427, "p05": 0.08152313530445099, "p95": 0.11384639143943787 }, { "label": "completed_group_4", "mean": 0.1107850894331932, "median": 0.1089654266834259, "p05": 0.08814431726932526, "p95": 0.139452263712883 }, { "label": "completed_group_5", "mean": 0.12967455387115479, "median": 0.1299237608909607, "p05": 0.10832332074642181, "p95": 0.15138982236385345 }, { "label": "completed_group_6", "mean": 0.29583215713500977, "median": 0.29647016525268555, "p05": 0.2276897132396698, "p95": 0.3604946434497833 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.5841834545135498, "group": 7, "latest_source_mass": 0.40767741203308105, "layer": 25, "max_source_mass": 0.40767741203308105, "mixer_index": 49, "normalized_entropy": 0.7618312045616817, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.07896879315376282, "median": 0.06809917092323303, "p05": 0.025932887569069862, "p95": 0.19368338584899902 }, { "label": "completed_group_1", "mean": 0.048752427101135254, "median": 0.04767637699842453, "p05": 0.034443892538547516, "p95": 0.0667153000831604 }, { "label": "completed_group_2", "mean": 0.055707164108753204, "median": 0.0549345538020134, "p05": 0.038696639239788055, "p95": 0.0750277042388916 }, { "label": "completed_group_3", "mean": 0.04139826446771622, "median": 0.03878612071275711, "p05": 0.026673054322600365, "p95": 0.05746251344680786 }, { "label": "completed_group_4", "mean": 0.04449018836021423, "median": 0.04196460545063019, "p05": 0.026162121444940567, "p95": 0.07164917886257172 }, { "label": "completed_group_5", "mean": 0.04285799711942673, "median": 0.04109475016593933, "p05": 0.029393533244729042, "p95": 0.06277710199356079 }, { "label": "completed_group_6", "mean": 0.2801477909088135, "median": 0.2763082981109619, "p05": 0.1455708146095276, "p95": 0.4254714846611023 }, { "label": "current_group_partial", "mean": 0.40767741203308105, "median": 0.4120897948741913, "p05": 0.21399420499801636, "p95": 0.5853031277656555 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.8125314712524414, "group": 7, "latest_source_mass": 0.3700707256793976, "layer": 26, "max_source_mass": 0.3700707256793976, "mixer_index": 50, "normalized_entropy": 0.8716433883436914, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.10889513790607452, "median": 0.11333495378494263, "p05": 0.04850132763385773, "p95": 0.1620434820652008 }, { "label": "completed_group_1", "mean": 0.08656036853790283, "median": 0.08640051633119583, "p05": 0.0650845319032669, "p95": 0.1084962785243988 }, { "label": "completed_group_2", "mean": 0.048123255372047424, "median": 0.04800213500857353, "p05": 0.03576180338859558, "p95": 0.06107594817876816 }, { "label": "completed_group_3", "mean": 0.06561170518398285, "median": 0.06379580497741699, "p05": 0.050968848168849945, "p95": 0.08332888036966324 }, { "label": "completed_group_4", "mean": 0.06917259097099304, "median": 0.06724178791046143, "p05": 0.050207968801259995, "p95": 0.09373342245817184 }, { "label": "completed_group_5", "mean": 0.08286689221858978, "median": 0.08183546364307404, "p05": 0.06535022705793381, "p95": 0.10340933501720428 }, { "label": "completed_group_6", "mean": 0.16869932413101196, "median": 0.16509854793548584, "p05": 0.11059704422950745, "p95": 0.23654375970363617 }, { "label": "current_group_partial", "mean": 0.3700707256793976, "median": 0.3694133758544922, "p05": 0.23441238701343536, "p95": 0.5025027990341187 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.50089693069458, "group": 7, "latest_source_mass": 0.49096518754959106, "layer": 26, "max_source_mass": 0.49096518754959106, "mixer_index": 51, "normalized_entropy": 0.7217788529328457, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.07974998652935028, "median": 0.07402504235506058, "p05": 0.04579866677522659, "p95": 0.1359037309885025 }, { "label": "completed_group_1", "mean": 0.04166398197412491, "median": 0.04038450866937637, "p05": 0.027992289513349533, "p95": 0.05969971418380737 }, { "label": "completed_group_2", "mean": 0.04607272148132324, "median": 0.044438350945711136, "p05": 0.030038591474294662, "p95": 0.06808309257030487 }, { "label": "completed_group_3", "mean": 0.03586764633655548, "median": 0.03317485377192497, "p05": 0.02203454077243805, "p95": 0.05572832003235817 }, { "label": "completed_group_4", "mean": 0.04223340004682541, "median": 0.03930673003196716, "p05": 0.02465122565627098, "p95": 0.06968094408512115 }, { "label": "completed_group_5", "mean": 0.04095568507909775, "median": 0.03919431194663048, "p05": 0.026239857077598572, "p95": 0.06058088690042496 }, { "label": "completed_group_6", "mean": 0.22249138355255127, "median": 0.2125401496887207, "p05": 0.10647299885749817, "p95": 0.3711881935596466 }, { "label": "current_group_partial", "mean": 0.49096518754959106, "median": 0.4988659620285034, "p05": 0.2878011167049408, "p95": 0.6676807403564453 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.8059929609298706, "group": 7, "latest_source_mass": 0.37139976024627686, "layer": 27, "max_source_mass": 0.37139976024627686, "mixer_index": 52, "normalized_entropy": 0.8684990295379666, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.10711923241615295, "median": 0.10972198843955994, "p05": 0.04896187782287598, "p95": 0.16201043128967285 }, { "label": "completed_group_1", "mean": 0.07651707530021667, "median": 0.07666032761335373, "p05": 0.05141128972172737, "p95": 0.10152198374271393 }, { "label": "completed_group_2", "mean": 0.058284737169742584, "median": 0.05812416970729828, "p05": 0.039273619651794434, "p95": 0.07745087146759033 }, { "label": "completed_group_3", "mean": 0.06609587371349335, "median": 0.06383530795574188, "p05": 0.04481930285692215, "p95": 0.09336265176534653 }, { "label": "completed_group_4", "mean": 0.07139429450035095, "median": 0.0694514587521553, "p05": 0.0485839769244194, "p95": 0.10034126043319702 }, { "label": "completed_group_5", "mean": 0.07493151724338531, "median": 0.07319007813930511, "p05": 0.051876723766326904, "p95": 0.10402432829141617 }, { "label": "completed_group_6", "mean": 0.17425751686096191, "median": 0.16937679052352905, "p05": 0.10449051856994629, "p95": 0.2624811828136444 }, { "label": "current_group_partial", "mean": 0.37139976024627686, "median": 0.3732905089855194, "p05": 0.23830299079418182, "p95": 0.5014542937278748 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.5622732639312744, "group": 7, "latest_source_mass": 0.4768889844417572, "layer": 27, "max_source_mass": 0.4768889844417572, "mixer_index": 53, "normalized_entropy": 0.7512946301290215, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.08775827288627625, "median": 0.08504067361354828, "p05": 0.0428270623087883, "p95": 0.144285649061203 }, { "label": "completed_group_1", "mean": 0.047765426337718964, "median": 0.04646281152963638, "p05": 0.031814731657505035, "p95": 0.06807165592908859 }, { "label": "completed_group_2", "mean": 0.053928617388010025, "median": 0.052262187004089355, "p05": 0.035437777638435364, "p95": 0.0784272700548172 }, { "label": "completed_group_3", "mean": 0.04293380305171013, "median": 0.03996016085147858, "p05": 0.02556057646870613, "p95": 0.06623870134353638 }, { "label": "completed_group_4", "mean": 0.04174989461898804, "median": 0.03903091698884964, "p05": 0.025013400241732597, "p95": 0.06869364529848099 }, { "label": "completed_group_5", "mean": 0.04734005779027939, "median": 0.04609068110585213, "p05": 0.029884062707424164, "p95": 0.06911227852106094 }, { "label": "completed_group_6", "mean": 0.20163494348526, "median": 0.19222964346408844, "p05": 0.09963390231132507, "p95": 0.3307305574417114 }, { "label": "current_group_partial", "mean": 0.4768889844417572, "median": 0.48383551836013794, "p05": 0.276212215423584, "p95": 0.6487575173377991 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.8757455348968506, "group": 7, "latest_source_mass": 0.3109493851661682, "layer": 28, "max_source_mass": 0.3109493851661682, "mixer_index": 54, "normalized_entropy": 0.9020429270551009, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.11563669890165329, "median": 0.12087491154670715, "p05": 0.05733206868171692, "p95": 0.1688953936100006 }, { "label": "completed_group_1", "mean": 0.08108697831630707, "median": 0.08185897022485733, "p05": 0.05175650864839554, "p95": 0.10753051191568375 }, { "label": "completed_group_2", "mean": 0.07280858606100082, "median": 0.07357072830200195, "p05": 0.04870978742837906, "p95": 0.0948689877986908 }, { "label": "completed_group_3", "mean": 0.07553227245807648, "median": 0.07362619787454605, "p05": 0.056317005306482315, "p95": 0.09756095707416534 }, { "label": "completed_group_4", "mean": 0.07606014609336853, "median": 0.07456053793430328, "p05": 0.05528895929455757, "p95": 0.1022825762629509 }, { "label": "completed_group_5", "mean": 0.06935927271842957, "median": 0.06843917071819305, "p05": 0.04978285729885101, "p95": 0.09167985618114471 }, { "label": "completed_group_6", "mean": 0.19856664538383484, "median": 0.19508783519268036, "p05": 0.1281973123550415, "p95": 0.27796900272369385 }, { "label": "current_group_partial", "mean": 0.3109493851661682, "median": 0.3105033040046692, "p05": 0.18172834813594818, "p95": 0.43916669487953186 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.6924852132797241, "group": 7, "latest_source_mass": 0.3315581679344177, "layer": 28, "max_source_mass": 0.3315581679344177, "mixer_index": 55, "normalized_entropy": 0.8139133413255193, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.08638577908277512, "median": 0.058608733117580414, "p05": 0.026544082909822464, "p95": 0.2740764319896698 }, { "label": "completed_group_1", "mean": 0.059912845492362976, "median": 0.05901291221380234, "p05": 0.03629481792449951, "p95": 0.08707985281944275 }, { "label": "completed_group_2", "mean": 0.06609142571687698, "median": 0.0647939145565033, "p05": 0.038691695779561996, "p95": 0.09703366458415985 }, { "label": "completed_group_3", "mean": 0.056745611131191254, "median": 0.05418796092271805, "p05": 0.032174739986658096, "p95": 0.08370660990476608 }, { "label": "completed_group_4", "mean": 0.06277773529291153, "median": 0.0600711889564991, "p05": 0.03501531109213829, "p95": 0.10094858705997467 }, { "label": "completed_group_5", "mean": 0.05352145433425903, "median": 0.05179818347096443, "p05": 0.032271772623062134, "p95": 0.08084158599376678 }, { "label": "completed_group_6", "mean": 0.2830069661140442, "median": 0.2759752869606018, "p05": 0.12981083989143372, "p95": 0.46204638481140137 }, { "label": "current_group_partial", "mean": 0.3315581679344177, "median": 0.3204721510410309, "p05": 0.13446375727653503, "p95": 0.5667697787284851 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.8980522155761719, "group": 8, "latest_source_mass": 0.20123262703418732, "layer": 29, "max_source_mass": 0.26602888107299805, "mixer_index": 56, "normalized_entropy": 0.9127701729200177, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.11603894829750061, "median": 0.09991352260112762, "p05": 0.030035395175218582, "p95": 0.2941743731498718 }, { "label": "completed_group_1", "mean": 0.08804173022508621, "median": 0.08880250155925751, "p05": 0.05664174258708954, "p95": 0.11779069900512695 }, { "label": "completed_group_2", "mean": 0.07975024729967117, "median": 0.07984748482704163, "p05": 0.05428915470838547, "p95": 0.10499341040849686 }, { "label": "completed_group_3", "mean": 0.09027000516653061, "median": 0.08898856490850449, "p05": 0.06189651042222977, "p95": 0.11996553838253021 }, { "label": "completed_group_4", "mean": 0.08430293202400208, "median": 0.0819794088602066, "p05": 0.05464346706867218, "p95": 0.12159879505634308 }, { "label": "completed_group_5", "mean": 0.07433462142944336, "median": 0.07287181913852692, "p05": 0.052520088851451874, "p95": 0.1010315865278244 }, { "label": "completed_group_6", "mean": 0.26602888107299805, "median": 0.2639757990837097, "p05": 0.15474572777748108, "p95": 0.3846370279788971 }, { "label": "completed_group_7", "mean": 0.20123262703418732, "median": 0.18140269815921783, "p05": 0.08419936150312424, "p95": 0.3782624900341034 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.7512003183364868, "group": 8, "latest_source_mass": 0.33482813835144043, "layer": 29, "max_source_mass": 0.33482813835144043, "mixer_index": 57, "normalized_entropy": 0.7970056117156376, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.07744461297988892, "median": 0.06970229744911194, "p05": 0.014401955530047417, "p95": 0.17144526541233063 }, { "label": "completed_group_1", "mean": 0.046086348593235016, "median": 0.04394831880927086, "p05": 0.028066720813512802, "p95": 0.0718703418970108 }, { "label": "completed_group_2", "mean": 0.05025433003902435, "median": 0.04861236736178398, "p05": 0.029644077643752098, "p95": 0.07761462777853012 }, { "label": "completed_group_3", "mean": 0.03748488053679466, "median": 0.03434944897890091, "p05": 0.020774798467755318, "p95": 0.05938909202814102 }, { "label": "completed_group_4", "mean": 0.04005061462521553, "median": 0.037498839199543, "p05": 0.019541850313544273, "p95": 0.06922487914562225 }, { "label": "completed_group_5", "mean": 0.041623204946517944, "median": 0.039552345871925354, "p05": 0.022081367671489716, "p95": 0.06821337342262268 }, { "label": "completed_group_6", "mean": 0.198423832654953, "median": 0.18318486213684082, "p05": 0.07548617571592331, "p95": 0.3721975088119507 }, { "label": "completed_group_7", "mean": 0.17380405962467194, "median": 0.1514919549226761, "p05": 0.05777408182621002, "p95": 0.36326974630355835 }, { "label": "current_group_partial", "mean": 0.33482813835144043, "median": 0.33602219820022583, "p05": 0.0795319601893425, "p95": 0.5979493856430054 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.0160694122314453, "group": 8, "latest_source_mass": 0.21711516380310059, "layer": 30, "max_source_mass": 0.21711516380310059, "mixer_index": 58, "normalized_entropy": 0.9175527313077866, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.09385271370410919, "median": 0.09211117774248123, "p05": 0.032377734780311584, "p95": 0.16496515274047852 }, { "label": "completed_group_1", "mean": 0.06208505108952522, "median": 0.06140740588307381, "p05": 0.0407925546169281, "p95": 0.0863042101264 }, { "label": "completed_group_2", "mean": 0.06560768187046051, "median": 0.06516212224960327, "p05": 0.041897669434547424, "p95": 0.09085705131292343 }, { "label": "completed_group_3", "mean": 0.07029052078723907, "median": 0.06780634075403214, "p05": 0.04846132919192314, "p95": 0.09917552024126053 }, { "label": "completed_group_4", "mean": 0.06935097277164459, "median": 0.06708167493343353, "p05": 0.04668639227747917, "p95": 0.09815650433301926 }, { "label": "completed_group_5", "mean": 0.056832700967788696, "median": 0.05595892667770386, "p05": 0.038822539150714874, "p95": 0.0771557092666626 }, { "label": "completed_group_6", "mean": 0.19425815343856812, "median": 0.19145764410495758, "p05": 0.12249615788459778, "p95": 0.27496030926704407 }, { "label": "completed_group_7", "mean": 0.17060703039169312, "median": 0.16317546367645264, "p05": 0.09239593893289566, "p95": 0.2726203501224518 }, { "label": "current_group_partial", "mean": 0.21711516380310059, "median": 0.21198657155036926, "p05": 0.11019522696733475, "p95": 0.3391069769859314 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 1.7837426662445068, "group": 8, "latest_source_mass": 0.29026859998703003, "layer": 30, "max_source_mass": 0.29026859998703003, "mixer_index": 59, "normalized_entropy": 0.8118162725118464, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.08836327493190765, "median": 0.07904411852359772, "p05": 0.02631576545536518, "p95": 0.17534467577934265 }, { "label": "completed_group_1", "mean": 0.04953558370471001, "median": 0.04658038914203644, "p05": 0.025181813165545464, "p95": 0.08318150043487549 }, { "label": "completed_group_2", "mean": 0.04813014343380928, "median": 0.045740917325019836, "p05": 0.026714935898780823, "p95": 0.07843323051929474 }, { "label": "completed_group_3", "mean": 0.0373263955116272, "median": 0.03401809185743332, "p05": 0.01990434154868126, "p95": 0.06290259957313538 }, { "label": "completed_group_4", "mean": 0.037015400826931, "median": 0.03405017405748367, "p05": 0.018646355718374252, "p95": 0.06544196605682373 }, { "label": "completed_group_5", "mean": 0.03390160948038101, "median": 0.03198609501123428, "p05": 0.018189728260040283, "p95": 0.05570102855563164 }, { "label": "completed_group_6", "mean": 0.1840541511774063, "median": 0.17327937483787537, "p05": 0.07966600358486176, "p95": 0.32982927560806274 }, { "label": "completed_group_7", "mean": 0.23140482604503632, "median": 0.22406929731369019, "p05": 0.08831030130386353, "p95": 0.40350067615509033 }, { "label": "current_group_partial", "mean": 0.29026859998703003, "median": 0.2745037376880646, "p05": 0.10280418395996094, "p95": 0.5399085879325867 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 1.9973840713500977, "group": 8, "latest_source_mass": 0.19101998209953308, "layer": 31, "max_source_mass": 0.20831164717674255, "mixer_index": 60, "normalized_entropy": 0.9090486661912383, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.10261696577072144, "median": 0.10389091819524765, "p05": 0.01371697522699833, "p95": 0.21163305640220642 }, { "label": "completed_group_1", "mean": 0.06039813905954361, "median": 0.06085003539919853, "p05": 0.026107093319296837, "p95": 0.09348798543214798 }, { "label": "completed_group_2", "mean": 0.06671924144029617, "median": 0.06656201928853989, "p05": 0.034684643149375916, "p95": 0.09819839894771576 }, { "label": "completed_group_3", "mean": 0.08377155661582947, "median": 0.0804532840847969, "p05": 0.05448959022760391, "p95": 0.12039553374052048 }, { "label": "completed_group_4", "mean": 0.07002593576908112, "median": 0.06737451255321503, "p05": 0.044841229915618896, "p95": 0.10655929893255234 }, { "label": "completed_group_5", "mean": 0.05768897384405136, "median": 0.056077782064676285, "p05": 0.039495617151260376, "p95": 0.08242568373680115 }, { "label": "completed_group_6", "mean": 0.20831164717674255, "median": 0.2021285891532898, "p05": 0.11652229726314545, "p95": 0.31706833839416504 }, { "label": "completed_group_7", "mean": 0.1594475358724594, "median": 0.1546279788017273, "p05": 0.06593266129493713, "p95": 0.2735573351383209 }, { "label": "current_group_partial", "mean": 0.19101998209953308, "median": 0.1713225245475769, "p05": 0.06408411264419556, "p95": 0.3850628137588501 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 1.969987154006958, "group": 8, "latest_source_mass": 0.16864418983459473, "layer": 31, "max_source_mass": 0.24503237009048462, "mixer_index": 61, "normalized_entropy": 0.8965797917640489, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.08120635151863098, "median": 0.061164144426584244, "p05": 0.0259944386780262, "p95": 0.23622018098831177 }, { "label": "completed_group_1", "mean": 0.07894632965326309, "median": 0.07658770680427551, "p05": 0.0496862456202507, "p95": 0.11596276611089706 }, { "label": "completed_group_2", "mean": 0.08615730702877045, "median": 0.08467075228691101, "p05": 0.050409119576215744, "p95": 0.12811316549777985 }, { "label": "completed_group_3", "mean": 0.07609779387712479, "median": 0.0722871869802475, "p05": 0.04099677503108978, "p95": 0.11950768530368805 }, { "label": "completed_group_4", "mean": 0.058728985488414764, "median": 0.056080836802721024, "p05": 0.03323862701654434, "p95": 0.09344407916069031 }, { "label": "completed_group_5", "mean": 0.05076971277594566, "median": 0.04946017265319824, "p05": 0.03047419898211956, "p95": 0.07563400268554688 }, { "label": "completed_group_6", "mean": 0.24503237009048462, "median": 0.24110376834869385, "p05": 0.10613486170768738, "p95": 0.3991503119468689 }, { "label": "completed_group_7", "mean": 0.1544169783592224, "median": 0.1400495171546936, "p05": 0.06199025735259056, "p95": 0.2995939254760742 }, { "label": "current_group_partial", "mean": 0.16864418983459473, "median": 0.15222924947738647, "p05": 0.0543956533074379, "p95": 0.3329208791255951 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.071648359298706, "group": 8, "latest_source_mass": 0.11520501971244812, "layer": 32, "max_source_mass": 0.20955611765384674, "mixer_index": 62, "normalized_entropy": 0.9428478002054053, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.10569609701633453, "median": 0.08689188957214355, "p05": 0.03173588588833809, "p95": 0.24571751058101654 }, { "label": "completed_group_1", "mean": 0.09922641515731812, "median": 0.09795533865690231, "p05": 0.06407676637172699, "p95": 0.14120742678642273 }, { "label": "completed_group_2", "mean": 0.08180510997772217, "median": 0.080217644572258, "p05": 0.054607540369033813, "p95": 0.11442367732524872 }, { "label": "completed_group_3", "mean": 0.10646799206733704, "median": 0.10510502755641937, "p05": 0.07415887713432312, "p95": 0.14216329157352448 }, { "label": "completed_group_4", "mean": 0.0826798751950264, "median": 0.08022409677505493, "p05": 0.05491747707128525, "p95": 0.11860857903957367 }, { "label": "completed_group_5", "mean": 0.06615597754716873, "median": 0.06485182791948318, "p05": 0.04477996751666069, "p95": 0.09238749742507935 }, { "label": "completed_group_6", "mean": 0.20955611765384674, "median": 0.206287682056427, "p05": 0.129981130361557, "p95": 0.2996031939983368 }, { "label": "completed_group_7", "mean": 0.13320739567279816, "median": 0.12465103715658188, "p05": 0.06303662061691284, "p95": 0.23065781593322754 }, { "label": "current_group_partial", "mean": 0.11520501971244812, "median": 0.10386130958795547, "p05": 0.03912060335278511, "p95": 0.22735844552516937 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 1.9781701564788818, "group": 8, "latest_source_mass": 0.16112206876277924, "layer": 32, "max_source_mass": 0.22251944243907928, "mixer_index": 63, "normalized_entropy": 0.9003040366848136, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.09446848928928375, "median": 0.074150450527668, "p05": 0.042592622339725494, "p95": 0.19443637132644653 }, { "label": "completed_group_1", "mean": 0.07056106626987457, "median": 0.06868506222963333, "p05": 0.04818308353424072, "p95": 0.0982733890414238 }, { "label": "completed_group_2", "mean": 0.08033448457717896, "median": 0.08047192543745041, "p05": 0.048296794295310974, "p95": 0.11253741383552551 }, { "label": "completed_group_3", "mean": 0.0658978745341301, "median": 0.06256154924631119, "p05": 0.04223886504769325, "p95": 0.0967884361743927 }, { "label": "completed_group_4", "mean": 0.058234501630067825, "median": 0.0563921295106411, "p05": 0.03529864549636841, "p95": 0.08846980333328247 }, { "label": "completed_group_5", "mean": 0.04316519945859909, "median": 0.042268507182598114, "p05": 0.029443470761179924, "p95": 0.05996086448431015 }, { "label": "completed_group_6", "mean": 0.2036968618631363, "median": 0.19645483791828156, "p05": 0.09398645162582397, "p95": 0.3395216464996338 }, { "label": "completed_group_7", "mean": 0.22251944243907928, "median": 0.2130982130765915, "p05": 0.11867167800664902, "p95": 0.35465875267982483 }, { "label": "current_group_partial", "mean": 0.16112206876277924, "median": 0.14023703336715698, "p05": 0.06008770316839218, "p95": 0.3247341811656952 } ], "sources": 9 }, { "branch": "output", "entropy_mean": 1.6036605834960938, "group": null, "latest_source_mass": 0.31639498472213745, "layer": null, "max_source_mass": 0.3412485122680664, "mixer_index": null, "normalized_entropy": 0.7298573846467136, "offset": null, "source_summaries": [ { "label": "embedding", "mean": 0.09759747982025146, "median": 0.09915056824684143, "p05": 0.025461740791797638, "p95": 0.16398483514785767 }, { "label": "completed_group_1", "mean": 0.04215940088033676, "median": 0.04050712287425995, "p05": 0.025720715522766113, "p95": 0.06398092210292816 }, { "label": "completed_group_2", "mean": 0.041145481169223785, "median": 0.03881259262561798, "p05": 0.023510465398430824, "p95": 0.0662468671798706 }, { "label": "completed_group_3", "mean": 0.029554028064012527, "median": 0.027420252561569214, "p05": 0.01627083122730255, "p95": 0.04924973472952843 }, { "label": "completed_group_4", "mean": 0.0289996936917305, "median": 0.027226522564888, "p05": 0.01560397818684578, "p95": 0.048351090401411057 }, { "label": "completed_group_5", "mean": 0.03647579252719879, "median": 0.0347352996468544, "p05": 0.020744025707244873, "p95": 0.058257561177015305 }, { "label": "completed_group_6", "mean": 0.06642461568117142, "median": 0.057809896767139435, "p05": 0.022837381809949875, "p95": 0.14078694581985474 }, { "label": "completed_group_7", "mean": 0.3412485122680664, "median": 0.3336990475654602, "p05": 0.0819513276219368, "p95": 0.6423859596252441 }, { "label": "completed_group_8", "mean": 0.31639498472213745, "median": 0.30713334679603577, "p05": 0.05841828137636185, "p95": 0.6156357526779175 } ], "sources": 9 } ], "mode": "uniform_except_group_6", "optimizer_state_after": "ed03e6fbd4a12d8b063dcb22e0437754285f54d585374cd52fbd534f05d24637", "optimizer_state_before": "ed03e6fbd4a12d8b063dcb22e0437754285f54d585374cd52fbd534f05d24637", "optimizer_state_unchanged": true, "positions": { "attention_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "5e6ecbcd78543fc42d4098a51169dca446d404ce384015643e84d2c78433c949", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.0933690071105957, 0.24537226557731628, 0.07698935270309448, 0.0523640513420105, 0.09380833059549332, 0.03647654876112938, 0.039120711386203766, 0.06371714919805527, 0.05156813934445381, 0.03728010877966881, 0.03983496502041817, 0.040284499526023865, 0.046871740370988846, 0.04314620420336723, 0.04033593460917473, 0.04471700266003609, 0.08397205173969269, 0.0528092160820961, 0.04443390294909477, 0.07560808956623077, 0.03708760067820549, 0.03599191829562187, 0.033304885029792786, 0.04061548039317131, 0.025487789884209633, 0.021565141156315804, 0.02342291921377182, 0.017244853079319, 0.024166326969861984, 0.016819356009364128, 0.021590016782283783, 0.026201345026493073 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 7.991803158802213e-06, "first_to_last_ratio": 2.1995682163489696, "imbalance_abs_log_ratio": 0.7882610758057936, "last_quartile_mean": 3.6333509001451603e-06, "mean": 5.700429795751916e-06, "normalized": [ 1.6446467361376822, 1.7533636838016493, 1.5121430301140257, 1.3642146592082889, 1.3427849100059275, 1.2710624599598845, 1.2093984224438232, 1.1181062389509275, 1.0508213097667785, 0.970116362120182, 0.9237211106568025, 0.8841286857317118, 0.8753694761779789, 0.8426547060112264, 0.8199057305029446, 0.7973604185829204, 0.8150774755026702, 0.7971502932826969, 0.8029407849463331, 0.8038150306430486, 1.7245493944356274, 1.4716789920415392, 1.176298502334216, 0.9296362096548916, 0.7451332726347899, 0.7284871846427164, 0.7031117226558635, 0.6604939800303086, 0.6232583249398803, 0.5847951838986024, 0.5382068389313539, 0.515568869252708 ], "peak_layer": 2, "peak_normalized": 1.7533636838016493, "population_cv": 0.34477524708616636, "rest_mean": 5.479317223944236e-06, "spike_contrast": 1.258265838923356, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.894437683513388e-06, "top_five_layers": [ 2, 21, 1, 3, 22 ], "top_five_spike_overlap": 2 }, "values": [ 9.375193258165382e-06, 9.994926585932262e-06, 8.619865184300579e-06, 7.776609891152475e-06, 7.654451110283844e-06, 7.245602319017053e-06, 6.894090802234132e-06, 6.373686119331978e-06, 5.990133104205597e-06, 5.530080215976341e-06, 5.265607342153089e-06, 5.039913503424032e-06, 4.989982244296698e-06, 4.803493993676966e-06, 4.673815055866726e-06, 4.5452970880432986e-06, 4.646291927201673e-06, 4.544099283521064e-06, 4.5771075747325085e-06, 4.582091150950873e-06, 9.830672752286773e-06, 8.389202776015736e-06, 6.70540703140432e-06, 5.299325948726619e-06, 4.247579909133492e-06, 4.152690053160768e-06, 4.008039013569942e-06, 3.7650995636795415e-06, 3.552840325937723e-06, 3.333583890707814e-06, 3.068010300921742e-06, 2.93896414405026e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 3.1796074836165644e-05, "first_to_last_ratio": 2.1837755018087943, "imbalance_abs_log_ratio": 0.7810552603870866, "last_quartile_mean": 1.456013899314712e-05, "mean": 2.27374501093891e-05, "normalized": [ 1.6525693591075148, 1.7573577785945664, 1.5181746471283126, 1.3599909015330387, 1.3342026751765357, 1.2583828704972015, 1.1984449281468548, 1.1080854962645417, 1.0413082187495306, 0.9637435404062689, 0.9174432285342764, 0.8810032008571269, 0.8753821812684542, 0.8435898167144781, 0.8218693355869752, 0.7997290159840107, 0.8200409422263285, 0.7990130185839762, 0.8020915674049959, 0.8039059608164798, 1.7323119495422332, 1.4806367034379706, 1.1809996314950881, 0.9268481543826393, 0.7452407338445852, 0.7290455126534822, 0.7045616815603938, 0.6634949906838151, 0.6274326816350748, 0.5895100193417423, 0.5431881075481845, 0.520401150293323 ], "peak_layer": 2, "peak_normalized": 1.7573577785945664, "population_cv": 0.34476107055991395, "rest_mean": 2.1839710589793633e-05, "spike_contrast": 1.2630773380346931, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.7585243515204638e-05, "top_five_layers": [ 2, 21, 1, 3, 22 ], "top_five_spike_overlap": 2 }, "values": [ 3.757521335501224e-05, 3.9957834815140814e-05, 3.451942029641941e-05, 3.0922725272830576e-05, 3.0336366762639955e-05, 2.8612417736439966e-05, 2.7249581762589514e-05, 2.519503868825268e-05, 2.3676693672314286e-05, 2.191307066823356e-05, 2.0860319636994973e-05, 2.0031766325701028e-05, 1.9903958673239686e-05, 1.918108137033414e-05, 1.8687213014345616e-05, 1.8183798601967283e-05, 1.8645640011527576e-05, 1.8167518646805547e-05, 1.82375169970328e-05, 1.827877167670522e-05, 3.93883565266151e-05, 3.3665903174551204e-05, 2.685292020032648e-05, 2.107416366925463e-05, 1.694487400527578e-05, 1.6576635971432552e-05, 1.6019936083466746e-05, 1.5086184248502832e-05, 1.4266219295677729e-05, 1.3403954653767869e-05, 1.2350712495390326e-05, 1.1832595191663131e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.0281970773357898, "first_to_last_ratio": 2.183775511771762, "imbalance_abs_log_ratio": 0.781055264949354, "last_quartile_mean": 0.012912076897919178, "mean": 0.020163798239082098, "normalized": [ 1.6525692790678699, 1.7573578810693697, 1.5181745396058195, 1.3599909305529354, 1.3342026811264212, 1.2583829165722413, 1.198444938084835, 1.108085514133287, 1.0413082229368562, 0.9637434812483201, 0.9174432981157873, 0.8810032057373075, 0.8753821438725333, 0.8435898428936303, 0.8218693577927582, 0.7997290250965632, 0.82004087301788, 0.7990129286039465, 0.8020915338425221, 0.8039059775097019, 1.7323120552124067, 1.4806366543070277, 1.18099969315099, 0.9268481613781459, 0.7452407515345857, 0.7290454885008755, 0.7045617226425356, 0.6634949929017351, 0.6274326709833373, 0.5895099946706145, 0.5431881032465743, 0.5204011405905861 ], "peak_layer": 2, "peak_normalized": 1.7573578810693697, "population_cv": 0.3447610793412406, "rest_mean": 0.01936767374475797, "spike_contrast": 1.2630773747442683, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.024462870508432388, "top_five_layers": [ 2, 21, 1, 3, 22 ], "top_five_spike_overlap": 2 }, "values": [ 0.03332207351922989, 0.03543500974774361, 0.030612165108323097, 0.027422582730650902, 0.026902593672275543, 0.025373779237270355, 0.02416520193219185, 0.022343212738633156, 0.020996728911995888, 0.019432729110121727, 0.018499141559004784, 0.017764370888471603, 0.0176510289311409, 0.017009975388646126, 0.016572007909417152, 0.016125574707984924, 0.016535138711333275, 0.016111135482788086, 0.016173211857676506, 0.0162097979336977, 0.03492999076843262, 0.029855258762836456, 0.023813439533114433, 0.018688779324293137, 0.015026884153485298, 0.014700326137244701, 0.014206640422344208, 0.013378579169511795, 0.012651425786316395, 0.011886760592460632, 0.01095273531973362, 0.010493263602256775 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 1.2478331257170794e-06, "first_to_last_ratio": 1.302648798616465, "imbalance_abs_log_ratio": 0.2643997289007993, "last_quartile_mean": 9.57919837674126e-07, "mean": 1.1239200041757158e-06, "normalized": [ 2.1469547613824664, 1.2055269606197945, 1.0904671862408197, 0.9797130509789345, 0.8878163931328654, 0.8592978812746958, 0.8501152961351623, 0.8621151687973471, 0.8387280509999393, 0.8036663656054353, 0.7892248336417454, 0.7888714083005081, 0.778767530084847, 0.790480788349666, 0.8030473654979075, 0.8180969225692613, 0.8074294250057539, 0.8357366807605537, 0.8650518157655221, 0.8971149996995309, 1.9079356509837764, 1.7922540948152808, 1.5351756980046967, 1.2479916121666939, 0.99020656697778, 0.9814067412805025, 0.9585141053894023, 0.8876551567293358, 0.8237812138581867, 0.7832469484624532, 0.7189681444212117, 0.6746411820679237 ], "peak_layer": 1, "peak_normalized": 2.1469547613824664, "population_cv": 0.3503982346959688, "rest_mean": 1.0209537953819058e-06, "spike_contrast": 1.6454589220992903, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.67993753166229e-06, "top_five_layers": [ 1, 21, 22, 23, 24 ], "top_five_spike_overlap": 4 }, "values": [ 2.4130054043780547e-06, 1.3549158666137373e-06, 1.2255978845132631e-06, 1.1011190963472473e-06, 9.97834604277159e-07, 9.657820783104398e-07, 9.554615871820715e-07, 9.689484841146623e-07, 9.426632345821417e-07, 9.032567049871432e-07, 8.870255783222092e-07, 8.866283565112099e-07, 8.752724056648731e-07, 8.884371709427796e-07, 9.025609983837057e-07, 9.194754966301844e-07, 9.074860827240627e-07, 9.393011737302004e-07, 9.722490403873962e-07, 1.008285494208394e-06, 2.144367044820683e-06, 2.014350229728734e-06, 1.725414676911896e-06, 1.402642737957649e-06, 1.1129129688924877e-06, 1.103022668758058e-06, 1.0772931773317396e-06, 9.976533874578308e-07, 9.258641853193694e-07, 8.803069135865371e-07, 8.08062679880095e-07, 7.582427201668906e-07 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 2.457950176903978e-05, "first_to_last_ratio": 2.4770306362250833, "imbalance_abs_log_ratio": 0.9070605187089287, "last_quartile_mean": 9.922970434672607e-06, "mean": 1.6918171411361982e-05, "normalized": [ 1.7073037028865254, 1.814747709956313, 1.5833333800919867, 1.4276250350274111, 1.402366518948633, 1.3146791729701672, 1.234921413660874, 1.1377930083834054, 1.0617236164497221, 0.9812405481092129, 0.9310402607494569, 0.8902341553187643, 0.8873157164893356, 0.848714352682437, 0.822336908188618, 0.7958645800274335, 0.8350535774705774, 0.8066537400213519, 0.8135540673730283, 0.815519637644919, 1.7540986073607638, 1.4633846195570983, 1.1347932867749078, 0.8434834940503287, 0.6822435655744704, 0.6624129328535199, 0.6402254758487317, 0.6055461674072661, 0.576037938199546, 0.5412548221877218, 0.5024963761847653, 0.4820016115507076 ], "peak_layer": 2, "peak_normalized": 1.814747709956313, "population_cv": 0.3757304181717806, "rest_mean": 1.6368015301985133e-05, "spike_contrast": 1.2151146021707844, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.9889014401996975e-05, "top_five_layers": [ 2, 21, 1, 3, 22 ], "top_five_spike_overlap": 2 }, "values": [ 2.8884456696687266e-05, 3.070221282541752e-05, 2.6787105525727384e-05, 2.4152805053745396e-05, 2.3725477149127983e-05, 2.2241967599256895e-05, 2.0892612155876122e-05, 1.9249377146479674e-05, 1.796242213458754e-05, 1.6600795788690448e-05, 1.5751498722238466e-05, 1.50611340359319e-05, 1.5011759387562051e-05, 1.4358694897964597e-05, 1.3912436770624481e-05, 1.3464573385135736e-05, 1.4127579561318271e-05, 1.3647106243297458e-05, 1.3763847164227627e-05, 1.379710101900855e-05, 2.967614091176074e-05, 2.475779183441773e-05, 1.9198627342120744e-05, 1.4270198334997986e-05, 1.154231358668767e-05, 1.1206815543118864e-05, 1.0831444342329632e-05, 1.0244733857689425e-05, 9.745508577907458e-06, 9.157041858998127e-06, 8.501319825882092e-06, 8.154585884767585e-06 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 1.843468248807767e-05, "first_to_last_ratio": 2.4175738950916372, "imbalance_abs_log_ratio": 0.8827645146172024, "last_quartile_mean": 7.6252819098954205e-06, "mean": 1.2676360086061322e-05, "normalized": [ 1.6871290330702846, 1.8098611247910557, 1.6119711458338681, 1.4453908307883032, 1.4329323413700108, 1.3225821129434003, 1.2121258419966898, 1.1120617380197748, 1.0366421166065487, 0.9623898238821091, 0.9155877639182657, 0.865219791037313, 0.8706927473808478, 0.8369629033014232, 0.810538513539737, 0.7856575518513865, 0.8421944299560972, 0.8136471808678093, 0.8301879485734135, 0.8333390185600643, 1.786035279002969, 1.447607822582003, 1.1093579407464549, 0.8076001743190514, 0.6770132591490092, 0.6567517478372727, 0.6378273910905584, 0.6102738422075373, 0.5949629309689786, 0.5661326746272846, 0.5403415983466012, 0.5289813808338772 ], "peak_layer": 2, "peak_normalized": 1.8098611247910557, "population_cv": 0.3740485667111722, "rest_mean": 1.2287799378369797e-05, "spike_contrast": 1.2023786727510586, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.477458790759556e-05, "top_five_layers": [ 2, 21, 1, 3, 22 ], "top_five_spike_overlap": 2 }, "values": [ 2.1386655134847388e-05, 2.2942451323615387e-05, 2.043392669293098e-05, 1.832229463616386e-05, 1.8164366338169202e-05, 1.6765527107054368e-05, 1.5365343642770313e-05, 1.4096895029069856e-05, 1.3140848750481382e-05, 1.2199599950690754e-05, 1.160632018581964e-05, 1.0967837624775711e-05, 1.1037214790121652e-05, 1.0609643140924163e-05, 1.0274678061250597e-05, 9.95927803160157e-06, 1.0675959856598638e-05, 1.0314084647689015e-05, 1.0523761375225149e-05, 1.0563705473032314e-05, 2.2640426323050633e-05, 1.8350398022448644e-05, 1.4062620721233543e-05, 1.0237430615234189e-05, 8.582063856010791e-06, 8.325221642735414e-06, 8.08532968221698e-06, 7.736050974926911e-06, 7.5419643508212175e-06, 7.176501640060451e-06, 6.849564670119435e-06, 6.705558462272165e-06 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 6.631883888985612e-05, "first_to_last_ratio": 2.132676882618221, "imbalance_abs_log_ratio": 0.7573779430719797, "last_quartile_mean": 3.109652448074485e-05, "mean": 4.7782829426523676e-05, "normalized": [ 1.6489229211468879, 1.708861354138125, 1.4921161329837158, 1.3497518288700419, 1.3272249524501374, 1.2676900386290495, 1.20321353768748, 1.1055950175375295, 1.0272590035437894, 0.952121373855485, 0.9135163903045121, 0.8885326150882444, 0.8870252045128123, 0.8604441024682924, 0.8384477395535956, 0.8227860946603986, 0.8309364966761863, 0.8093959581523735, 0.8099057627492265, 0.8094529076503494, 1.6551141233880609, 1.4612953387676968, 1.178654371228179, 0.9454272953654747, 0.7596541500480443, 0.7487371292388807, 0.7206557733740219, 0.6750022996845395, 0.6401201992245045, 0.5945928823774631, 0.549322789967469, 0.5182242146774328 ], "peak_layer": 2, "peak_normalized": 1.708861354138125, "population_cv": 0.32976388398829903, "rest_mean": 4.601283789843028e-05, "spike_contrast": 1.246190982716678, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.7340783678228034e-05, "top_five_layers": [ 2, 21, 1, 3, 22 ], "top_five_spike_overlap": 2 }, "values": [ 7.879020267864689e-05, 8.16542305983603e-05, 7.129753066692501e-05, 6.449496140703559e-05, 6.341856351355091e-05, 6.0573816881515086e-05, 5.7492947235004976e-05, 5.282845813781023e-05, 4.908534174319357e-05, 4.549505320028402e-05, 4.365039785625413e-05, 4.24566023866646e-05, 4.238457404426299e-05, 4.1114453779300675e-05, 4.006340532214381e-05, 3.931504761567339e-05, 3.970449688495137e-05, 3.867522900691256e-05, 3.8699588913004845e-05, 3.8677950215060264e-05, 7.908603583928198e-05, 6.982482591411099e-05, 5.6319440773222595e-05, 4.5175191189628094e-05, 3.629842467489652e-05, 3.577677853172645e-05, 3.443497189437039e-05, 3.225351974833757e-05, 3.058675429201685e-05, 2.8411330276867375e-05, 2.624819717311766e-05, 2.476221925462596e-05 ] } } }, "mlp_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "c0b8018c349b1bd09e5d17858eb312a2323da0e92de9651e2bb08ae4adf115c7", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.13583384454250336, 0.0415157787501812, 0.018140379339456558, 0.039518751204013824, 0.022877270355820656, 0.015532293356955051, 0.02314678393304348, 0.05432315170764923, 0.03743875026702881, 0.04006272554397583, 0.045583780854940414, 0.05995568633079529, 0.038651175796985626, 0.04559725895524025, 0.05632574483752251, 0.07952121645212173, 0.042337141931056976, 0.05044656619429588, 0.04418695345520973, 0.06310147047042847, 0.029010163620114326, 0.04586305841803551, 0.06033914536237717, 0.07656749337911606, 0.045884523540735245, 0.051604680716991425, 0.06502768397331238, 0.06792232394218445, 0.060306549072265625, 0.08025261014699936, 0.09663844108581543, 0.12230555713176727 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 7.786419416788704e-06, "first_to_last_ratio": 2.2517035065758906, "imbalance_abs_log_ratio": 0.8116870437838105, "last_quartile_mean": 3.458012741930361e-06, "mean": 5.418445589100429e-06, "normalized": [ 1.5689064519991014, 1.818827228999633, 1.5778512631742794, 1.39281528032429, 1.4035696994799445, 1.3238828290319984, 1.2489526208375292, 1.161361619065899, 1.0532204633332511, 0.985703672342169, 0.9405176049676927, 0.9039748873146388, 0.903831542057675, 0.86911647643091, 0.84443515916264, 0.8246952248214869, 0.8511666792859153, 0.832159836759504, 0.8409305019111457, 0.8528501467972953, 1.6301467685245534, 1.2765548914567177, 1.0049904913189498, 0.7839965857518106, 0.7663754420179811, 0.7396131007499015, 0.6956205001357612, 0.6560371929117643, 0.6150250909997227, 0.566862679249974, 0.5430944746255953, 0.5229135941602691 ], "peak_layer": 2, "peak_normalized": 1.818827228999633, "population_cv": 0.33811487932989037, "rest_mean": 5.325717084916606e-06, "spike_contrast": 1.111433337016204, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.919179511693074e-06, "top_five_layers": [ 2, 21, 3, 1, 5 ], "top_five_spike_overlap": 1 }, "values": [ 8.501034244545735e-06, 9.855216376308817e-06, 8.549501217203215e-06, 7.546893812104827e-06, 7.60516604714212e-06, 7.1733870754542295e-06, 6.767381819372531e-06, 6.292774742178153e-06, 5.706817773898365e-06, 5.3409817155625205e-06, 5.096143468108494e-06, 4.898138740827562e-06, 4.897362032352248e-06, 4.709260338131571e-06, 4.575525963446125e-06, 4.468566203286173e-06, 4.612000338966027e-06, 4.509012796916068e-06, 4.5565361688204575e-06, 4.6211221160774585e-06, 8.832861567498185e-06, 6.916943220858229e-06, 5.445486294775037e-06, 4.2480428419366945e-06, 4.1525636333972216e-06, 4.007553343399195e-06, 3.7691818306484492e-06, 3.5547018342185766e-06, 3.3324799915135372e-06, 3.071514584007673e-06, 2.942727860499872e-06, 2.833378857758362e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 3.095534998465155e-05, "first_to_last_ratio": 2.2311980934257, "imbalance_abs_log_ratio": 0.8025387028245473, "last_quartile_mean": 1.3873868965674774e-05, "mean": 2.1617158267872583e-05, "normalized": [ 1.56962397447842, 1.8219373390106741, 1.5830595072374052, 1.388205659387903, 1.39393219024475, 1.3100820795919315, 1.2378283306608768, 1.151175244923606, 1.0448112136253231, 0.9798330324557321, 0.9355503848650201, 0.901787034930718, 0.9047451745570857, 0.8717787678855152, 0.8471475748953301, 0.8278330393977466, 0.8562509540339507, 0.8334378115278304, 0.8392781072765801, 0.8517625419784799, 1.642374603355249, 1.2867275441017108, 1.0025500806522716, 0.7838963986380294, 0.7669509833560733, 0.740932060634184, 0.6981125793325658, 0.6603083038036954, 0.6196109752933442, 0.5717337105301445, 0.5482485810219997, 0.5284942163158535 ], "peak_layer": 2, "peak_normalized": 1.8219373390106741, "population_cv": 0.3371032431078189, "rest_mean": 2.1230851955435895e-05, "spike_contrast": 1.1164513042050481, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.3703212355030702e-05, "top_five_layers": [ 2, 21, 3, 1, 5 ], "top_five_spike_overlap": 1 }, "values": [ 3.39308098773472e-05, 3.9385107811540365e-05, 3.422124791541137e-05, 3.000906144734472e-05, 3.0132852771203034e-05, 2.832025165844243e-05, 2.6758330932352692e-05, 2.4885137463570572e-05, 2.2585849364986643e-05, 2.1181205738685094e-05, 2.0223940737196244e-05, 1.9494073058012873e-05, 1.9558019630494528e-05, 1.884537959995214e-05, 1.8312923202756792e-05, 1.7895397832035087e-05, 1.8509712390368804e-05, 1.801655707822647e-05, 1.8142807675758377e-05, 1.8412685676594265e-05, 3.550347173586488e-05, 2.781539296847768e-05, 2.167228376492858e-05, 1.6945612514973618e-05, 1.6579300790908746e-05, 1.601684562047012e-05, 1.5091210116224829e-05, 1.4273989108914975e-05, 1.339422851742711e-05, 1.2359258107608184e-05, 1.1851576346089132e-05, 1.1424543117755093e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.02745151426643133, "first_to_last_ratio": 2.231198158290128, "imbalance_abs_log_ratio": 0.8025387318961152, "last_quartile_mean": 0.012303485535085201, "mean": 0.019170312298228964, "normalized": [ 1.569624039062625, 1.8219373190372243, 1.5830595450647444, 1.3882056485060128, 1.393932144530304, 1.3100821230068005, 1.2378283118012143, 1.1511752347471484, 1.0448112818515607, 0.9798330364586034, 0.9355504139626032, 0.9017870470373622, 0.9047451747238671, 0.8717787387654422, 0.8471475283519182, 0.8278330775706999, 0.8562509246141621, 0.8334378283497555, 0.8392781022528738, 0.8517625767622847, 1.6423746447126553, 1.2867275373156608, 1.0025500385110884, 0.7838964039539179, 0.7669509815600329, 0.7409319919464318, 0.6981125957238729, 0.6603083165846706, 0.6196109603907951, 0.5717336953626899, 0.5482485711743644, 0.5284941663066132 ], "peak_layer": 2, "peak_normalized": 1.8219373190372243, "population_cv": 0.3371032525044943, "rest_mean": 0.01882773199705062, "spike_contrast": 1.1164513032098002, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.021020245924592017, "top_five_layers": [ 2, 21, 3, 1, 5 ], "top_five_spike_overlap": 1 }, "values": [ 0.03009018301963806, 0.03492710739374161, 0.03034774586558342, 0.026612335816025734, 0.02672211453318596, 0.02511468343436718, 0.023729555308818817, 0.022068388760089874, 0.020029358565807343, 0.018783705309033394, 0.01793479360640049, 0.017287539318203926, 0.017344247549772263, 0.01671227067708969, 0.016240082681179047, 0.01586981862783432, 0.016414597630500793, 0.01597726345062256, 0.016089223325252533, 0.01632855460047722, 0.03148483484983444, 0.02466696873307228, 0.01921919733285904, 0.015027538873255253, 0.014702689833939075, 0.014203897677361965, 0.013383036479353905, 0.012658316642045975, 0.011878135614097118, 0.010960313491523266, 0.010510096326470375, 0.01013139821588993 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 1.1928892789114798e-06, "first_to_last_ratio": 1.3156875444213576, "imbalance_abs_log_ratio": 0.27435937646132796, "last_quartile_mean": 9.066660879852861e-07, "mean": 1.0454644723267847e-06, "normalized": [ 2.066766454588109, 1.2579770052809085, 1.1414476795366335, 0.9991135550715228, 0.9502780977311038, 0.9106312522579444, 0.9003664662888449, 0.9015290366240919, 0.8668956194037676, 0.8358138540139084, 0.8219636515441451, 0.8237021242735056, 0.8237910759650436, 0.8321372565922732, 0.8460983224477613, 0.865523936473377, 0.8625090939647664, 0.8923621733366707, 0.9258956561316818, 0.9664554526149962, 1.7395990883104075, 1.5105028105825653, 1.268487006487683, 1.05225262139031, 1.0447177171624922, 1.0189469549691033, 0.9644907946553192, 0.8920953182620569, 0.8305679878140533, 0.768540929372819, 0.7322664730608089, 0.6862745337913257 ], "peak_layer": 1, "peak_normalized": 2.066766454588109, "population_cv": 0.28778640653741316, "rest_mean": 9.829085540870977e-07, "spike_contrast": 1.4073195569101953, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.383266430821095e-06, "top_five_layers": [ 1, 21, 22, 23, 2 ], "top_five_spike_overlap": 3 }, "values": [ 2.160730900868657e-06, 1.315170266025234e-06, 1.1933429959753994e-06, 1.0445377256473876e-06, 9.934819900081493e-07, 9.52032621626131e-07, 9.413011525793991e-07, 9.425165785614809e-07, 9.063085713023611e-07, 8.73813689850067e-07, 8.593337952333968e-07, 8.611513067080523e-07, 8.612443025413086e-07, 8.699699378666992e-07, 8.845657362144266e-07, 9.048745255313406e-07, 9.017226147989277e-07, 9.329329486718052e-07, 9.679910135673708e-07, 1.010394839795481e-06, 1.818689042920596e-06, 1.579177023813827e-06, 1.3261580988910282e-06, 1.1000927315762965e-06, 1.092215256903728e-06, 1.0652728406057577e-06, 1.0083408596983645e-06, 9.326539611720364e-07, 8.683293231115385e-07, 8.03482237188291e-07, 7.655585818611144e-07, 7.174756433414586e-07 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 2.4038813990046037e-05, "first_to_last_ratio": 2.5264760675230087, "imbalance_abs_log_ratio": 0.9268254731235277, "last_quartile_mean": 9.514760222373297e-06, "mean": 1.6183600706654033e-05, "normalized": [ 1.6431319593536555, 1.8709304858170968, 1.6443718115023969, 1.4565107761776974, 1.4580036341452054, 1.3619970862849295, 1.270276913326693, 1.1778258221812326, 1.0624446034984056, 0.9942000210888526, 0.9458438765376493, 0.9079496518490903, 0.9126111519933036, 0.8730322637265189, 0.8434173200758008, 0.819763355367018, 0.8673363170905152, 0.8380086603668375, 0.847411406380904, 0.8624822807545881, 1.674865475495682, 1.2829417034878756, 0.9639272173505105, 0.7173079128565587, 0.6975950392314004, 0.6714721041459226, 0.6357268553578129, 0.6042829864814293, 0.5666055757530662, 0.528258222251376, 0.5081459456155611, 0.4913215644544134 ], "peak_layer": 2, "peak_normalized": 1.8709304858170968, "population_cv": 0.37248317331628894, "rest_mean": 1.598182276480926e-05, "spike_contrast": 1.0808029751556287, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.7273201592615806e-05, "top_five_layers": [ 2, 21, 3, 1, 5 ], "top_five_spike_overlap": 1 }, "values": [ 2.6591791538521647e-05, 3.027839193237014e-05, 2.6611856810632162e-05, 2.35715888265986e-05, 2.3595748643856496e-05, 2.204201700806152e-05, 2.055765435216017e-05, 1.9061462808167562e-05, 1.7194179235957563e-05, 1.608973616384901e-05, 1.530715962871909e-05, 1.469389462727122e-05, 1.476933448429918e-05, 1.412880556017626e-05, 1.364952913718298e-05, 1.3266722817206755e-05, 1.4036624634172767e-05, 1.356199754809495e-05, 1.3714167835132685e-05, 1.3958068848296534e-05, 2.7105354092782363e-05, 2.0762616259162314e-05, 1.5599813195876777e-05, 1.1608624845393933e-05, 1.128959956986364e-05, 1.0866836419154424e-05, 1.0288349585607648e-05, 9.77947456703987e-06, 9.169718396151438e-06, 8.549120138923172e-06, 8.223631084547378e-06, 7.95135201769881e-06 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 1.8185147837357363e-05, "first_to_last_ratio": 2.4574101026437742, "imbalance_abs_log_ratio": 0.8991079915537216, "last_quartile_mean": 7.4001274015245144e-06, "mean": 1.2241909388421846e-05, "normalized": [ 1.653346118631287, 1.8535740256527908, 1.6603698292615041, 1.4733741948539911, 1.4789521533278296, 1.3622205194815116, 1.2413453867006845, 1.160681480185816, 1.0374732332878815, 0.9706557097325654, 0.9220125389142853, 0.8803571206561456, 0.8900958141641991, 0.8526598971250481, 0.8248525705777248, 0.8002108914807246, 0.8672492154167579, 0.8421108097566772, 0.8593593907763116, 0.8805185605038235, 1.7078847041908367, 1.2915152153330152, 0.9513392425094038, 0.7019111860511013, 0.6831283692770284, 0.6607641584407571, 0.6345355694742685, 0.6178894347154805, 0.5870244092215966, 0.5595527408573421, 0.5501686503833274, 0.5428668590582837 ], "peak_layer": 2, "peak_normalized": 1.8535740256527908, "population_cv": 0.37132218564122205, "rest_mean": 1.2089665957588565e-05, "spike_contrast": 1.0805942828156805, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.3064023914921562e-05, "top_five_layers": [ 2, 21, 3, 1, 5 ], "top_five_spike_overlap": 1 }, "values": [ 2.024011337198317e-05, 2.2691285266773775e-05, 2.0326097001088783e-05, 1.803691338864155e-05, 1.8105198250850663e-05, 1.66761801665416e-05, 1.5196437743725255e-05, 1.4208957509254105e-05, 1.2700653314823285e-05, 1.1882679245900363e-05, 1.128719395637745e-05, 1.0777252100524493e-05, 1.0896472304011695e-05, 1.043818519974593e-05, 1.0097770427819341e-05, 9.796109225135297e-06, 1.0616786312311888e-05, 1.0309044228051789e-05, 1.0520199793973006e-05, 1.0779228432511445e-05, 2.090776979457587e-05, 1.58106122398749e-05, 1.1646208804449998e-05, 8.592733138357289e-06, 8.36279559734976e-06, 8.089014954748563e-06, 7.767926945234649e-06, 7.564146471850108e-06, 7.186299626482651e-06, 6.849993951618671e-06, 6.735114766343031e-06, 6.645726898568682e-06 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 6.394726369762793e-05, "first_to_last_ratio": 2.165750659168782, "imbalance_abs_log_ratio": 0.7727670261485797, "last_quartile_mean": 2.9526604748753016e-05, "mean": 4.527678021304382e-05, "normalized": [ 1.5496786655198094, 1.7638183513650956, 1.548338431381764, 1.3622589737794613, 1.3827812688433938, 1.3140688787417427, 1.2397136851001251, 1.138248318877889, 1.0299800591585466, 0.9735369180024122, 0.9413007515926131, 0.9215371993923233, 0.9187706765142037, 0.8922886784234674, 0.8669509413854829, 0.8549741655971893, 0.8687534277415242, 0.8524543165082663, 0.850237305696699, 0.8571636164381721, 1.5722187039695086, 1.2626116014185806, 1.0198765578723143, 0.8013529083096489, 0.7893707490868679, 0.7608525271636171, 0.7140148794287507, 0.6768580931979752, 0.6280524787589078, 0.5803286132527243, 0.5470014576866675, 0.5206067997942558 ], "peak_layer": 2, "peak_normalized": 1.7638183513650956, "population_cv": 0.319309976956315, "rest_mean": 4.452982985067474e-05, "spike_contrast": 1.1073546055575072, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.9310312169836835e-05, "top_five_layers": [ 2, 21, 1, 3, 5 ], "top_five_spike_overlap": 1 }, "values": [ 7.016446033958346e-05, 7.986001583049074e-05, 7.010377885308117e-05, 6.16787001490593e-05, 6.26078835921362e-05, 5.949680780759081e-05, 5.6130244047380984e-05, 5.15362189617008e-05, 4.663418076233938e-05, 4.407861706567928e-05, 4.26190672442317e-05, 4.172423723503016e-05, 4.1598977986723185e-05, 4.039995837956667e-05, 3.925274722860195e-05, 3.871047738357447e-05, 3.9334358007181436e-05, 3.859638673020527e-05, 3.849600761895999e-05, 3.880960866808891e-05, 7.118500070646405e-05, 5.7166987971868366e-05, 4.617672675522044e-05, 3.628267950261943e-05, 3.574016591301188e-05, 3.4448952646926045e-05, 3.232829476473853e-05, 3.064595512114465e-05, 2.8436194043024443e-05, 2.627541107358411e-05, 2.4766464775893837e-05, 2.357139965170063e-05 ] } } }, "post_attention_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "90a633f12d984935028fe5d7212be48e7da7b4a0806751939a551938d16dc540", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.0933690071105957, 0.35791492462158203, 0.3940892517566681, 0.4057151973247528, 0.09380833059549332, 0.11286329478025436, 0.13231655955314636, 0.1621972620487213, 0.05156813934445381, 0.08533531427383423, 0.12003090232610703, 0.16063299775123596, 0.046871740370988846, 0.079384446144104, 0.11459959298372269, 0.15625107288360596, 0.08397205173969269, 0.11654024571180344, 0.15294702351093292, 0.21572710573673248, 0.03708760067820549, 0.06553087383508682, 0.09390194714069366, 0.12852425873279572, 0.025487789884209633, 0.06059965491294861, 0.08860157430171967, 0.12100546807050705, 0.024166326969861984, 0.06877554953098297, 0.12682415544986725, 0.2025238573551178 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 7.991837151166692e-06, "first_to_last_ratio": 2.199555600045736, "imbalance_abs_log_ratio": 0.7882553399803155, "last_quartile_mean": 3.633387194668103e-06, "mean": 5.700454799750787e-06, "normalized": [ 1.6446163877737412, 1.7533998686231527, 1.5121451725020296, 1.364197826084461, 1.3427906671116336, 1.271051699375988, 1.2094097106049357, 1.1181073176234888, 1.050800346888206, 0.9701192865312583, 0.9237230419640569, 0.8841126022610825, 0.8753754487166554, 0.842645186368349, 0.8199032509711276, 0.7973532515096635, 0.8150810001894991, 0.7971392182113938, 0.8029398955317023, 0.8038076757100767, 1.7245582634337944, 1.4716851410634708, 1.1763049099281253, 0.9296488844893543, 0.7451497881646588, 0.7284882970564497, 0.7031155789121764, 0.6604896868519163, 0.6232609359810303, 0.5848060208123226, 0.5382108600955737, 0.5155627786586261 ], "peak_layer": 2, "peak_normalized": 1.7533998686231527, "population_cv": 0.34477630009007615, "rest_mean": 5.479330571621109e-06, "spike_contrast": 1.2582788246724164, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.894525631651049e-06, "top_five_layers": [ 2, 21, 1, 3, 22 ], "top_five_spike_overlap": 2 }, "values": [ 9.375061381433625e-06, 9.99517669697525e-06, 8.619915206509177e-06, 7.776548045512754e-06, 7.654517503397074e-06, 7.245572760439245e-06, 6.894185389683116e-06, 6.373720225383295e-06, 5.9900398809986655e-06, 5.5301211432379205e-06, 5.265641448204406e-06, 5.039843927079346e-06, 4.990038178220857e-06, 4.803460797120351e-06, 4.673821422329638e-06, 4.545276169665158e-06, 4.646332399715902e-06, 4.544056082522729e-06, 4.577122581395088e-06, 4.582069323078031e-06, 9.830766430241056e-06, 8.389274626097176e-06, 6.705472969770199e-06, 5.299421445670305e-06, 4.247692686476512e-06, 4.152714609517716e-06, 4.008078576589469e-06, 3.765091605600901e-06, 3.552870794010232e-06, 3.333660288262763e-06, 3.068046680709813e-06, 2.9389423161774175e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 3.179608893333352e-05, "first_to_last_ratio": 2.1837668191217623, "imbalance_abs_log_ratio": 0.7810512843811803, "last_quartile_mean": 1.456020333989727e-05, "mean": 2.2737495811497865e-05, "normalized": [ 1.652548917558419, 1.7573782461899687, 1.5181795955693564, 1.3599736080488836, 1.3341926334858063, 1.258380741159644, 1.198450039243716, 1.1080873490022425, 1.0412974057810096, 0.963746803263346, 0.9174525044210594, 0.880993030098069, 0.8753893817096504, 0.8435885211062553, 0.821862483668264, 0.799722208567091, 0.8200429739325733, 0.7990069326021415, 0.8020937951883779, 0.8038991450039482, 1.7323316674839808, 1.4806342073722776, 1.1810094976301715, 0.9268510914012806, 0.7452497958595891, 0.7290454072725963, 0.7045664253646529, 0.6634924970729509, 0.627430380508366, 0.5895163943906402, 0.5431853357557247, 0.5204009842879473 ], "peak_layer": 2, "peak_normalized": 1.7573782461899687, "population_cv": 0.34476205735593485, "rest_mean": 2.183972157110003e-05, "spike_contrast": 1.2630873804796743, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.758547670964617e-05, "top_five_layers": [ 2, 21, 1, 3, 22 ], "top_five_spike_overlap": 2 }, "values": [ 3.757482409127988e-05, 3.995838051196188e-05, 3.4519602195359766e-05, 3.092239421675913e-05, 3.033619941561483e-05, 2.8612426831386983e-05, 2.7249752747593448e-05, 2.519513145671226e-05, 2.36764954024693e-05, 2.191318890254479e-05, 2.0860572476522066e-05, 2.0031575331813656e-05, 1.9904162400052883e-05, 1.918109046528116e-05, 1.8687094780034386e-05, 1.8183680367656052e-05, 1.864572368504014e-05, 1.816741678339895e-05, 1.823760430852417e-05, 1.827865344239399e-05, 3.9388884033542126e-05, 3.3665914088487625e-05, 2.6853198505705222e-05, 2.1074272808618844e-05, 1.6945114111877047e-05, 1.6576666894252412e-05, 1.602007614565082e-05, 1.508615787315648e-05, 1.4266195648815483e-05, 1.3404126548266504e-05, 1.2350674296612851e-05, 1.183261520054657e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.028197089908644557, "first_to_last_ratio": 2.1837668773420766, "imbalance_abs_log_ratio": 0.7810513110416795, "last_quartile_mean": 0.01291213370859623, "mean": 0.020163838693406433, "normalized": [ 1.6525489664534274, 1.7573781882008161, 1.5181796227672884, 1.3599736990719704, 1.3341926142992153, 1.2583807614047304, 1.1984500160850322, 1.1080873555255117, 1.0412974504808812, 0.9637468131130092, 0.9174524501540883, 0.8809930320247955, 0.8753894404113371, 0.8435885199175384, 0.8218625358626679, 0.7997222475861995, 0.8200430151819952, 0.799006891534459, 0.8020938043935768, 0.8038991916191983, 1.732331488837583, 1.4806341456116123, 1.181009517302933, 0.9268511053875943, 0.7452497871819499, 0.7290453652759271, 0.7045664058783442, 0.663492507050367, 0.6274303498586399, 0.5895163867408781, 0.5431853506992427, 0.5204009740871907 ], "peak_layer": 2, "peak_normalized": 1.7573781882008161, "population_cv": 0.34476204573376257, "rest_mean": 0.019367683678865433, "spike_contrast": 1.2630873251313288, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.024463075771927835, "top_five_layers": [ 2, 21, 1, 3, 22 ], "top_five_spike_overlap": 2 }, "values": [ 0.03332173079252243, 0.03543549031019211, 0.03061232902109623, 0.027422290295362473, 0.026902444660663605, 0.025373786687850952, 0.024165352806448936, 0.022343294695019722, 0.02099655382335186, 0.019432835280895233, 0.018499363213777542, 0.017764201387763023, 0.017651211470365524, 0.017009982839226723, 0.016571903601288795, 0.016125470399856567, 0.016535215079784393, 0.016111046075820923, 0.016173290088772774, 0.016209693625569344, 0.034930452704429626, 0.029855268076062202, 0.023813685402274132, 0.018688876181840897, 0.01502709649503231, 0.014700353145599365, 0.014206763356924057, 0.01337855588644743, 0.012651404365897179, 0.01188691332936287, 0.010952701792120934, 0.010493281297385693 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 1.247834788387081e-06, "first_to_last_ratio": 1.3026493651654052, "imbalance_abs_log_ratio": 0.26440016382141457, "last_quartile_mean": 9.579206974308363e-07, "mean": 1.1239152044595357e-06, "normalized": [ 2.1469457225713144, 1.2055458656022202, 1.090459805968128, 0.9797143014525316, 0.8878314125094404, 0.8592517839018609, 0.8501357178957297, 0.8621718544003517, 0.838780135441054, 0.803587510126235, 0.7892340708971455, 0.7888778117764692, 0.7787236681934099, 0.7904303004147828, 0.8030666758808169, 0.8180409386059743, 0.8074943233056451, 0.8357557261342219, 0.865044686679264, 0.8971318795311857, 1.9079116323881797, 1.7922556795440105, 1.5352240300064353, 1.2479291695771983, 0.9902668341743711, 0.9814166981024416, 0.9585695842271079, 0.887627185602954, 0.8237575723950213, 0.7832528727327069, 0.7189648421901128, 0.6745997077716791 ], "peak_layer": 1, "peak_normalized": 2.1469457225713144, "population_cv": 0.35039894955314266, "rest_mean": 1.0209484478899093e-06, "spike_contrast": 1.6454657366957175, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.6799356899355189e-06, "top_five_layers": [ 1, 21, 22, 23, 24 ], "top_five_spike_overlap": 4 }, "values": [ 2.4129849407472648e-06, 1.3549313280236674e-06, 1.2255843557795743e-06, 1.1011157994289533e-06, 9.978472235161462e-07, 9.657261443862808e-07, 9.554804591971333e-07, 9.690080560176284e-07, 9.427177474208293e-07, 9.031642207446566e-07, 8.87032172158797e-07, 8.866317671163415e-07, 8.752193707550759e-07, 8.883766327016929e-07, 9.02578847217228e-07, 9.19408648769604e-07, 9.075551474779786e-07, 9.393185678163718e-07, 9.7223687589576e-07, 1.0083001598104602e-06, 2.1443308924062876e-06, 2.014343408518471e-06, 1.7254616295758751e-06, 1.4025665677763755e-06, 1.1129759514005855e-06, 1.1030291489078081e-06, 1.0773509302453022e-06, 9.976176897907862e-07, 9.258336604034412e-07, 8.80309812600899e-07, 8.080555176093185e-07, 7.581928684885497e-07 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 2.457953223711229e-05, "first_to_last_ratio": 2.477024426714826, "imbalance_abs_log_ratio": 0.9070580118695099, "last_quartile_mean": 9.923007610268542e-06, "mean": 1.69181895444126e-05, "normalized": [ 1.7072952069439127, 1.814751785835374, 1.583335876217373, 1.427616301262686, 1.4023620054109303, 1.3146851825449262, 1.234932562009686, 1.1377929715724469, 1.0617163500289413, 0.9812457323814809, 0.9310441011085459, 0.8902292230375227, 0.8873181522360982, 0.8487116152382174, 0.8223310272706993, 0.7958632431878757, 0.8350525749366983, 0.8066485210135078, 0.8135577648627865, 0.8155141941009584, 1.754100812941828, 1.4633856314926994, 1.1347958335819939, 0.8434818911388621, 0.6822498229316093, 0.6624116852901087, 0.6402281764287867, 0.6055451420692906, 0.5760378583814265, 0.541258865288379, 0.5024958376051377, 0.48200405164921045 ], "peak_layer": 2, "peak_normalized": 1.814751785835374, "population_cv": 0.3757302663581963, "rest_mean": 1.6368026316976518e-05, "spike_contrast": 1.2151172406131192, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.9889070972567425e-05, "top_five_layers": [ 2, 21, 1, 3, 22 ], "top_five_spike_overlap": 2 }, "values": [ 2.8884343919344246e-05, 3.070231468882412e-05, 2.6787176466314122e-05, 2.415268318145536e-05, 2.3725426217424683e-05, 2.224209310952574e-05, 2.0892823158646934e-05, 1.9249397155363113e-05, 1.796231845219154e-05, 1.6600901290075853e-05, 1.5751580576761626e-05, 1.5061066733323969e-05, 1.5011816685728263e-05, 1.4358663975144736e-05, 1.3912352187617216e-05, 1.346456519968342e-05, 1.4127577742328867e-05, 1.3647032574226614e-05, 1.3763924471277278e-05, 1.37970237119589e-05, 2.9676210033358075e-05, 2.4757835490163416e-05, 1.919869100674987e-05, 1.4270186511566862e-05, 1.15424318209989e-05, 1.1206806448171847e-05, 1.0831501640495844e-05, 1.0244727491226513e-05, 9.745517672854476e-06, 9.15712007554248e-06, 8.501319825882092e-06, 8.154635906976182e-06 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 1.8434964204061544e-05, "first_to_last_ratio": 2.417736714515905, "imbalance_abs_log_ratio": 0.8828318606214098, "last_quartile_mean": 7.624884915458097e-06, "mean": 1.2676840526637534e-05, "normalized": [ 1.6873428874794156, 1.8101714876361856, 1.6117740258736184, 1.4453888558243493, 1.432931412528032, 1.322431545876559, 1.211931679225086, 1.1118191375253939, 1.0366731385217558, 0.9625966360926657, 0.9155597362322775, 0.8653246778689294, 0.8705477556259337, 0.8369716471261985, 0.810751152519636, 0.7854979184496037, 0.8422636714323503, 0.8135084405288281, 0.830247313863565, 0.833295454432844, 1.7864035100286402, 1.4475196700904036, 1.1095320635464954, 0.8076642698727924, 0.6770119941301986, 0.6566109182967833, 0.6379494334906077, 0.6102109668893928, 0.5949331720909589, 0.5659879615181939, 0.5402230450459562, 0.5289244203363497 ], "peak_layer": 2, "peak_normalized": 1.8101714876361856, "population_cv": 0.3740992470966407, "rest_mean": 1.2288022339311687e-05, "spike_contrast": 1.2025091044084812, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.4776458738197106e-05, "top_five_layers": [ 2, 21, 1, 3, 22 ], "top_five_spike_overlap": 2 }, "values": [ 2.1390176698332652e-05, 2.294725527463015e-05, 2.043220229097642e-05, 1.8322964024264365e-05, 1.816504300222732e-05, 1.6764253814471886e-05, 1.536346462671645e-05, 1.4094353900873102e-05, 1.314174005528912e-05, 1.2202684047224466e-05, 1.1606404768826906e-05, 1.0969582945108414e-05, 1.1035795068892185e-05, 1.0610156095935963e-05, 1.0277763067279011e-05, 9.957631846191362e-06, 1.0677242244128138e-05, 1.0312716767657548e-05, 1.0524912795517594e-05, 1.0563553587417118e-05, 2.2645952412858605e-05, 1.834997601690702e-05, 1.4065361028769985e-05, 1.0238631148240529e-05, 8.582373084209394e-06, 8.32375189929735e-06, 8.087183232419193e-06, 7.735547114862129e-06, 7.541872946603689e-06, 7.174939128162805e-06, 6.848321390862111e-06, 6.705090527248103e-06 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 6.631942187595996e-05, "first_to_last_ratio": 2.1323970004075212, "imbalance_abs_log_ratio": 0.757246699291085, "last_quartile_mean": 3.110087936875061e-05, "mean": 4.778498180257884e-05, "normalized": [ 1.6490535968634175, 1.7087192127819313, 1.492007812264596, 1.3497840657052835, 1.327091931099493, 1.2676384196659913, 1.203093106343722, 1.1055851115936601, 1.0270846021796025, 0.9524010599086087, 0.9134150983529511, 0.8885677355109032, 0.8870934342358626, 0.8605623302629707, 0.8382236777547796, 0.8227225399283683, 0.8309637051450267, 0.8094418756876959, 0.8099509721310832, 0.8094557317595811, 1.6552119356260317, 1.4609820878747368, 1.1788822853535004, 0.9452636602424558, 0.7596605876204491, 0.7487585236759859, 0.7207180975870248, 0.675256249599339, 0.6401292801959797, 0.5946508734410685, 0.5493593713920817, 0.5182710282158182 ], "peak_layer": 2, "peak_normalized": 1.7087192127819313, "population_cv": 0.3297300915843088, "rest_mean": 4.601516667645666e-05, "spike_contrast": 1.246153988462618, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.7341983483638616e-05, "top_five_layers": [ 2, 21, 1, 3, 22 ], "top_five_spike_overlap": 2 }, "values": [ 7.879999611759558e-05, 8.165111648850143e-05, 7.129556615836918e-05, 6.449940701713786e-05, 6.341506377793849e-05, 6.0574078815989196e-05, 5.7489782193442807e-05, 5.2830364438705146e-05, 4.907921902486123e-05, 4.551046731648967e-05, 4.364752385299653e-05, 4.24601930717472e-05, 4.238974361214787e-05, 4.11219552916009e-05, 4.0054503188002855e-05, 3.9313781599048525e-05, 3.970758552895859e-05, 3.867916529998183e-05, 3.8703492464264855e-05, 3.867982741212472e-05, 7.909427222330123e-05, 6.981300248298794e-05, 5.633286855299957e-05, 4.516940680332482e-05, 3.630036735557951e-05, 3.5779412428382784e-05, 3.443950117798522e-05, 3.2267107599182054e-05, 3.058856600546278e-05, 2.841538116626907e-05, 2.6251127565046772e-05, 2.4765571652096696e-05 ] } } }, "post_mlp_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "89f2226456bfe96e245d2aad0f699d32c676f9f53099789cd870253d002402f5", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.16915453970432281, 0.374446302652359, 0.39966297149658203, 0.4194087088108063, 0.09892481565475464, 0.11664740741252899, 0.13951337337493896, 0.18593810498714447, 0.06607726216316223, 0.10425819456577301, 0.14551182091236115, 0.19865906238555908, 0.06484340131282806, 0.10430748760700226, 0.14720691740512848, 0.20439592003822327, 0.10052042454481125, 0.14085541665554047, 0.1745898574590683, 0.2420409768819809, 0.049456026405096054, 0.08509506285190582, 0.11783231794834137, 0.15649692714214325, 0.05533855780959129, 0.08507220447063446, 0.11942370235919952, 0.1559232622385025, 0.06628894805908203, 0.12384890764951706, 0.19928614795207977, 0.30097496509552 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 7.7864465310995e-06, "first_to_last_ratio": 2.2517152155576943, "imbalance_abs_log_ratio": 0.8116922438251685, "last_quartile_mean": 3.45800680179309e-06, "mean": 5.418447727834064e-06, "normalized": [ 1.568888544023948, 1.8188344001058179, 1.5778853856405257, 1.3928090236097381, 1.4035521085426979, 1.323910001979022, 1.2489444066892168, 1.1613786172193286, 1.0532471556339933, 0.9856901908530037, 0.9405250388286469, 0.903975957241629, 0.9038305978229731, 0.8691056426588686, 0.8444251743902574, 0.8247027043990806, 0.8511561883019437, 0.8321528781597636, 0.8409254701420178, 0.8528477120219263, 1.6301323612590062, 1.276560682015566, 1.0049909338934035, 0.7839975351843149, 0.7663664951659469, 0.7396123052601714, 0.6956139311329645, 0.656042808768423, 0.6150196028810266, 0.5668674490845093, 0.5430953093309747, 0.5229133877592945 ], "peak_layer": 2, "peak_normalized": 1.8188344001058179, "population_cv": 0.3381164045173533, "rest_mean": 5.3257224324086025e-06, "spike_contrast": 1.1114293691146353, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.919164323131554e-06, "top_five_layers": [ 2, 21, 3, 1, 5 ], "top_five_spike_overlap": 1 }, "values": [ 8.500940566591453e-06, 9.8552591225598e-06, 8.549689482606482e-06, 7.5468628892849665e-06, 7.60507373342989e-06, 7.173537142080022e-06, 6.7673399826162495e-06, 6.292869329627138e-06, 5.706964657292701e-06, 5.340910774975782e-06, 5.096185759612126e-06, 4.898146471532527e-06, 4.897358849120792e-06, 4.70920349471271e-06, 4.5754736675007734e-06, 4.468608494789805e-06, 4.61194531453657e-06, 4.5089768718753476e-06, 4.556510702968808e-06, 4.621110747393686e-06, 8.83278698893264e-06, 6.916977326909546e-06, 5.445490842248546e-06, 4.248049663146958e-06, 4.15251679442008e-06, 4.00755061491509e-06, 3.7691477245971328e-06, 3.5547336665331386e-06, 3.332451569804107e-06, 3.0715416414750507e-06, 2.942733544841758e-06, 2.833378857758362e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 3.095529586971679e-05, "first_to_last_ratio": 2.231190719147604, "imbalance_abs_log_ratio": 0.8025353977436115, "last_quartile_mean": 1.3873890566173941e-05, "mean": 2.161713939585752e-05, "normalized": [ 1.569631234977724, 1.821920754111271, 1.5830649282606075, 1.3882016542740292, 1.393923982845026, 1.3100899549651668, 1.237822932080286, 1.1511788584298446, 1.0448339194949339, 0.9798235379398699, 0.935556082062235, 0.9017856344130347, 0.9047411681050822, 0.871774732653454, 0.8471420035372855, 0.8278340145427652, 0.8562520381338441, 0.8334393805858055, 0.8392739595253591, 0.8517584051263162, 1.6423719981736806, 1.286730939363875, 1.0025528912413784, 0.7838911086437838, 0.7669532516816032, 0.7409356525761478, 0.6981128101373316, 0.6603089223334052, 0.6196116424395146, 0.5717387114554894, 0.5482482181923758, 0.5284946776974742 ], "peak_layer": 2, "peak_normalized": 1.821920754111271, "population_cv": 0.3371023410152076, "rest_mean": 2.1230832957102125e-05, "spike_contrast": 1.1164514464896436, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.3703194165136666e-05, "top_five_layers": [ 2, 21, 3, 1, 5 ], "top_five_spike_overlap": 1 }, "values": [ 3.393093720660545e-05, 3.93847149098292e-05, 3.422133522690274e-05, 3.00089486700017e-05, 3.0132649044389836e-05, 2.8320397177594714e-05, 2.675819087016862e-05, 2.4885193852242082e-05, 2.258632048324216e-05, 2.1180982002988458e-05, 2.022404623858165e-05, 1.949402576428838e-05, 1.9557915948098525e-05, 1.8845275917556137e-05, 1.8312786778551526e-05, 1.7895403289003298e-05, 1.850971966632642e-05, 1.8016575268120505e-05, 1.814270217437297e-05, 1.841258017520886e-05, 3.550338442437351e-05, 2.7815442081191577e-05, 2.167232560168486e-05, 1.6945483366725966e-05, 1.6579335351707414e-05, 1.6016909285099246e-05, 1.5091201930772513e-05, 1.4273990018409677e-05, 1.3394231245911215e-05, 1.2359355423541274e-05, 1.1851558156195097e-05, 1.1424543117755093e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.027451465604826808, "first_to_last_ratio": 2.2311907409001925, "imbalance_abs_log_ratio": 0.8025354074929298, "last_quartile_mean": 0.012303504627197981, "mean": 0.019170295243384317, "normalized": [ 1.5696311681003479, 1.8219206732617643, 1.5830649371197136, 1.388201636715615, 1.393924056984641, 1.3100899927740872, 1.2378229031074972, 1.151178882293153, 1.0448338787368396, 0.9798236088789388, 0.9355561044291842, 0.9017856145607054, 0.9047412186385304, 0.8717747533515304, 0.8471420635784898, 0.8278340083782124, 0.8562520750297825, 0.8334393471220807, 0.8392739907616938, 0.8517583792148204, 1.642372025000689, 1.2867309168033454, 1.0025528736911131, 0.7838910772353376, 0.7669532184870957, 0.7409356631720833, 0.6981128281470679, 0.6603089040276711, 0.6196116087905021, 0.5717387220890688, 0.5482482330371852, 0.5284946364812135 ], "peak_layer": 2, "peak_normalized": 1.8219206732617643, "population_cv": 0.33710233697872305, "rest_mean": 0.01882771488830999, "spike_contrast": 1.1164514273496355, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.021020229160785674, "top_five_layers": [ 2, 21, 3, 1, 5 ], "top_five_spike_overlap": 1 }, "values": [ 0.030090292915701866, 0.03492675721645355, 0.03034782223403454, 0.026612235233187675, 0.026721935719251633, 0.025114811956882477, 0.02372943051159382, 0.022068439051508904, 0.02002977393567562, 0.018783507868647575, 0.01793488673865795, 0.017287496477365494, 0.01734415628015995, 0.01671217940747738, 0.016239963471889496, 0.01586982235312462, 0.01641460508108139, 0.015977278351783752, 0.01608913019299507, 0.016328459605574608, 0.031484756618738174, 0.024667011573910713, 0.019219234585762024, 0.015027423389256, 0.014702719636261463, 0.014203955419361591, 0.013383029028773308, 0.012658316642045975, 0.011878137476742268, 0.010960400104522705, 0.010510080493986607, 0.01013139821588993 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 1.1928609140454682e-06, "first_to_last_ratio": 1.315659229091292, "imbalance_abs_log_ratio": 0.2743378549117157, "last_quartile_mean": 9.066640416222072e-07, "mean": 1.0454568002415954e-06, "normalized": [ 2.066856002243532, 1.2578654226981267, 1.1414899840844595, 0.9990468326089935, 0.9502783292381165, 0.910640436030826, 0.9003011396728565, 0.901481335018573, 0.8669278621175863, 0.8357944328654023, 0.8219569605121918, 0.8237561249823732, 0.823764715734452, 0.8320513660640659, 0.8460590766752437, 0.8655475783690212, 0.86251047564991, 0.8923630672646186, 0.9258984273005659, 0.966451561835344, 1.7396355604850735, 1.5105514119836407, 1.2685636276287062, 1.052272305153328, 1.0447386505549785, 1.0189836845663283, 0.9645164677436621, 0.8920489610952551, 0.8305323797297215, 0.7685686986531484, 0.7322675514226048, 0.6862795700172946 ], "peak_layer": 1, "peak_normalized": 2.066856002243532, "population_cv": 0.28780510099505935, "rest_mean": 9.82893507423594e-07, "spike_contrast": 1.4073738111107978, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.3832985814588028e-06, "top_five_layers": [ 1, 21, 22, 23, 2 ], "top_five_spike_overlap": 3 }, "values": [ 2.1608086626656586e-06, 1.3150439599485253e-06, 1.1933784662687685e-06, 1.0444603049108991e-06, 9.934749414242106e-07, 9.520352364233986e-07, 9.412259487362462e-07, 9.424597919860389e-07, 9.063356287697388e-07, 8.737869734432024e-07, 8.593204938733834e-07, 8.612014426034875e-07, 8.612104238636675e-07, 8.698737588019867e-07, 8.845182151162589e-07, 9.048926017385384e-07, 9.017174420478113e-07, 9.329270369562437e-07, 9.679868071543751e-07, 1.010383357424871e-06, 1.8187138266512193e-06, 1.579216245772841e-06, 1.3262284710435779e-06, 1.1001052371284459e-06, 1.09222912669793e-06, 1.0653034223651048e-06, 1.008360300147615e-06, 9.325986525254848e-07, 8.682857242092723e-07, 8.035053724597674e-07, 7.655540912310244e-07, 7.174756433414586e-07 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 2.4038814444793388e-05, "first_to_last_ratio": 2.52647246263142, "imbalance_abs_log_ratio": 0.9268240462767798, "last_quartile_mean": 9.514773978480662e-06, "mean": 1.6183600735075743e-05, "normalized": [ 1.6431375763217857, 1.8709298081489039, 1.6443663011578147, 1.456511897590528, 1.4579991357016089, 1.362001916967257, 1.270275787125066, 1.1778262697010342, 1.0624571901050601, 0.9941950738714855, 0.9458499443186641, 0.9079497064530831, 0.9126080032724408, 0.8730316440093773, 0.8434173185945877, 0.8197656018688684, 0.8673344610155396, 0.8380103448512647, 0.8474074147964744, 0.8624810990705939, 1.6748537832583608, 1.2829405772640068, 0.9639274404518096, 0.7173066190304442, 0.6975958247858137, 0.6714738451213601, 0.6357277534179556, 0.6042818614494239, 0.5666064739346, 0.528261761831544, 0.5081460009216912, 0.4913215635915523 ], "peak_layer": 2, "peak_normalized": 1.8709298081489039, "population_cv": 0.3724823338114802, "rest_mean": 1.5981830647096677e-05, "spike_contrast": 1.0807997901855262, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.72731592101627e-05, "top_five_layers": [ 2, 21, 3, 1, 5 ], "top_five_spike_overlap": 1 }, "values": [ 2.6591882487991825e-05, 3.027838101843372e-05, 2.6611767680151388e-05, 2.3571607016492635e-05, 2.3595675884280354e-05, 2.2042095224605873e-05, 2.0557636162266135e-05, 1.9061470084125176e-05, 1.719438296277076e-05, 1.6089656128315255e-05, 1.5307257854146883e-05, 1.4693895536765922e-05, 1.476928355259588e-05, 1.4128795555734541e-05, 1.364952913718298e-05, 1.3266759196994826e-05, 1.4036594620847609e-05, 1.3562024832936004e-05, 1.3714103261008859e-05, 1.3958049748907797e-05, 2.7105164917884395e-05, 2.076259806926828e-05, 1.5599816833855584e-05, 1.1608603927015793e-05, 1.1289612302789465e-05, 1.086686461349018e-05, 1.0288364137522876e-05, 9.779456377145834e-06, 9.169732948066667e-06, 8.549177437089384e-06, 8.22363199404208e-06, 7.95135201769881e-06 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 1.818386806462513e-05, "first_to_last_ratio": 2.457357195654607, "imbalance_abs_log_ratio": 0.8990864617493757, "last_quartile_mean": 7.3997659342239785e-06, "mean": 1.2241763258202809e-05, "normalized": [ 1.6532278156307396, 1.8529637577064542, 1.6601917288005896, 1.4734992122881876, 1.4787212184853242, 1.3624052800849313, 1.241630190592572, 1.1605300301836057, 1.0377105811344711, 0.970302362281516, 0.9221226537562405, 0.8804234989039083, 0.8901502729669281, 0.8528310713547791, 0.8248197718653548, 0.8003387203338942, 0.8672804445232961, 0.842284978427344, 0.8594742555132325, 0.8806659958909823, 1.7078242589221275, 1.2914181504288738, 0.9515274936827194, 0.7019045573087637, 0.6829698814124192, 0.6607754635415474, 0.6345934412628262, 0.6177168322437578, 0.5870025160165216, 0.5596171470160981, 0.5502030781671085, 0.5428733392728858 ], "peak_layer": 2, "peak_normalized": 1.8529637577064542, "population_cv": 0.3712553637233459, "rest_mean": 1.2089582570398408e-05, "spike_contrast": 1.0805616237183344, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.3063538972346578e-05, "top_five_layers": [ 2, 21, 3, 1, 5 ], "top_five_spike_overlap": 1 }, "values": [ 2.0238423530827276e-05, 2.2683543647872284e-05, 2.032367410720326e-05, 1.8038228517980315e-05, 1.810215508157853e-05, 1.667824290052522e-05, 1.5199742847471498e-05, 1.420693388354266e-05, 1.2703407264780253e-05, 1.1878211807925254e-05, 1.1288407222309615e-05, 1.0777936040540226e-05, 1.0897008905885741e-05, 1.0440156074764673e-05, 1.0097248377860524e-05, 9.797557140700519e-06, 1.0617041880323086e-05, 1.0311053301848006e-05, 1.0521480362513103e-05, 1.0780904631246813e-05, 2.090678026434034e-05, 1.5809235264896415e-05, 1.164837431133492e-05, 8.592549420427531e-06, 8.360755600733683e-06, 8.089056791504845e-06, 7.76854267314775e-06, 7.5619432209350634e-06, 7.185945833043661e-06, 6.850700629001949e-06, 6.735455826856196e-06, 6.645726898568682e-06 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 6.395047284968314e-05, "first_to_last_ratio": 2.1660681808857896, "imbalance_abs_log_ratio": 0.7729136258699902, "last_quartile_mean": 2.9523758030336467e-05, "mean": 4.527745574023356e-05, "normalized": [ 1.54980804632547, 1.763644033614954, 1.5487230192406263, 1.3621213403384314, 1.3824803823160707, 1.3138427773713444, 1.240068729413229, 1.1386166882660989, 1.0297248516588071, 0.9734193862329693, 0.9412691113074699, 0.9215307620305427, 0.9189040065908425, 0.8921894731007584, 0.8670235779425224, 0.8550896459349258, 0.8687420731506561, 0.8528263874244194, 0.8500407828624786, 0.8572385684307896, 1.5722309216636912, 1.2626846020158011, 1.019761387973834, 0.801514665952234, 0.789113185617719, 0.7606239932693007, 0.7140948597096711, 0.6768786878149542, 0.6278451696872341, 0.5804073741531897, 0.5469424761050102, 0.5205990324839538 ], "peak_layer": 2, "peak_normalized": 1.763644033614954, "population_cv": 0.3193228576879087, "rest_mean": 4.453070512142774e-05, "spike_contrast": 1.1073237880991371, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.930990908178501e-05, "top_five_layers": [ 2, 21, 1, 3, 5 ], "top_five_spike_overlap": 1 }, "values": [ 7.017136522335932e-05, 7.985331467352808e-05, 7.012223795754835e-05, 6.167338870000094e-05, 6.259519432205707e-05, 5.948745820205659e-05, 5.614715701085515e-05, 5.155366670805961e-05, 4.6623321395600215e-05, 4.407395317684859e-05, 4.261827052687295e-05, 4.1724568291101605e-05, 4.160563548794016e-05, 4.039606938022189e-05, 3.92566216760315e-05, 3.871628359775059e-05, 3.933443076675758e-05, 3.861380901071243e-05, 3.848768392344937e-05, 3.881358134094626e-05, 7.11866159690544e-05, 5.717114618164487e-05, 4.617220110958442e-05, 3.629054481280036e-05, 3.5729037335840985e-05, 3.4439119190210477e-05, 3.233239840483293e-05, 3.064734482904896e-05, 2.8427231882233173e-05, 2.6279369194526225e-05, 2.4764163754298352e-05, 2.357139965170063e-05 ] } } }, "pre_attention_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "ee695e78937de01b7128d3866b8fc5dd328ebdf2c159be37918d4f02952d5248", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.0626930519938469, 0.09030644595623016, 0.20229491591453552, 0.20300470292568207, 0.19049881398677826, 0.12695962190628052, 0.12132706493139267, 0.11585728824138641, 0.13427142798900604, 0.10485932976007462, 0.10419609397649765, 0.10852954536676407, 0.1286265105009079, 0.09096142649650574, 0.10080574452877045, 0.10566592961549759, 0.1287592649459839, 0.09650235623121262, 0.1065765842795372, 0.11446042358875275, 0.12395420670509338, 0.07221997529268265, 0.07677769660949707, 0.08416415005922318, 0.09094838052988052, 0.059439923614263535, 0.062182098627090454, 0.06896103918552399, 0.0776243805885315, 0.06285123527050018, 0.07231374084949493, 0.08016443997621536 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 6.401589445204081e-06, "first_to_last_ratio": 5.014158816962473, "imbalance_abs_log_ratio": 1.612265673937794, "last_quartile_mean": 1.276702569441568e-06, "mean": 3.973329489426192e-06, "normalized": [ 3.594355074839489, 1.980257395705191, 1.4331604383132732, 1.2175713013817075, 0.5122482645969012, 1.1161930258645065, 1.4866597228542398, 1.5486735112286358, 1.0211563074802648, 0.9723621747249844, 0.7560475968021131, 0.6769014524583965, 0.4622835091675935, 0.9151822371020745, 0.7406376553155025, 0.6916875845069519, 0.6765551269177021, 1.1975266520520074, 1.093455827073347, 1.3383663538174226, 1.3116030347782306, 2.132906386388474, 1.4299501172899827, 1.1237146764046368, 0.3251623148042092, 0.4145012283968733, 0.3058268259608037, 0.4017901597313135, 0.27069801691963735, 0.20880032371058876, 0.29396085570650865, 0.34980484770643766 ], "peak_layer": 1, "peak_normalized": 3.594355074839489, "population_cv": 0.6824649856288003, "rest_mean": 3.778586820620452e-06, "spike_contrast": 1.3298463524921948, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.0249399009771875e-06, "top_five_layers": [ 1, 22, 2, 8, 7 ], "top_five_spike_overlap": 1 }, "values": [ 1.4281557014328428e-05, 7.868215107009746e-06, 5.694418632629095e-06, 4.837811957258964e-06, 2.0353311356302584e-06, 4.4350026655592956e-06, 5.90698891755892e-06, 6.153390131657943e-06, 4.057390469824895e-06, 3.863515303237364e-06, 3.004026211783639e-06, 2.689552502488368e-06, 1.8368046994510223e-06, 3.6363205708767055e-06, 2.9427974368445575e-06, 2.748302676991443e-06, 2.6881764370045858e-06, 4.7581679609720595e-06, 4.344660283095436e-06, 5.317770501278574e-06, 5.21143101650523e-06, 8.474739843222778e-06, 5.681662969436729e-06, 4.464888661459554e-06, 1.2919770142616471e-06, 1.646949954192678e-06, 1.215150746247673e-06, 1.5964446902216878e-06, 1.0755724133559852e-06, 8.296324836010172e-07, 1.1680033367156284e-06, 1.3898899169362267e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 2.483370565187215e-05, "first_to_last_ratio": 4.981584561843075, "imbalance_abs_log_ratio": 1.6057480255354664, "last_quartile_mean": 4.985101696775018e-06, "mean": 1.565249977630856e-05, "normalized": [ 3.2972929755857625, 2.0048272210961917, 1.4248147397393138, 1.2525605952975924, 0.5136155031697731, 1.1389996412922376, 1.4837943897791828, 1.576613576696709, 0.993097921000044, 0.967178091525582, 0.7807637782290343, 0.6845730256942221, 0.46020229198505896, 0.9301610029487104, 0.7362785192832098, 0.6928493259141912, 0.689547719193988, 1.2669417970797459, 1.1062817709837498, 1.3390347891556382, 1.3317341947494643, 2.144680169638977, 1.4856156453191776, 1.150653491985306, 0.3162484388912845, 0.42389449598018586, 0.3025615965942938, 0.3760596062041592, 0.27322836561724756, 0.21350954230535987, 0.29113624537197047, 0.35124953169263623 ], "peak_layer": 1, "peak_normalized": 3.2972929755857625, "population_cv": 0.6538528737987491, "rest_mean": 1.4824116188147283e-05, "spike_contrast": 1.3576371701991343, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.0125771152379456e-05, "top_five_layers": [ 1, 22, 2, 8, 23 ], "top_five_spike_overlap": 2 }, "values": [ 5.161087756277993e-05, 3.1380557629745454e-05, 2.230191239505075e-05, 1.9605704437708482e-05, 8.039366548473481e-06, 1.7828191630542278e-05, 2.3225091354106553e-05, 2.4677943656570278e-05, 1.5544464986305684e-05, 1.513875486125471e-05, 1.2220904864079785e-05, 1.0715279131545685e-05, 7.203316272352822e-06, 1.4559344890585635e-05, 1.1524599358381238e-05, 1.0844823918887414e-05, 1.0793145520437974e-05, 1.9830806195386685e-05, 1.731607517285738e-05, 2.0959241737728007e-05, 2.084496918541845e-05, 3.3569605875527486e-05, 2.3253598556038924e-05, 1.8010603525908664e-05, 4.9500786190037616e-06, 6.635008503508288e-06, 4.735845323011745e-06, 5.886272901989287e-06, 4.276706931705121e-06, 3.3419580631743884e-06, 4.557010015560081e-06, 5.497933216247475e-06 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.022022778750397265, "first_to_last_ratio": 4.981584625420242, "imbalance_abs_log_ratio": 1.6057480382979046, "last_quartile_mean": 0.004420838027726859, "mean": 0.013880793398129754, "normalized": [ 3.297293025983657, 2.004827112271471, 1.424814718920783, 1.252560619837325, 0.5136155059381478, 1.1389997039452073, 1.4837943949140509, 1.5766136345126096, 0.9930978434697787, 0.9671780949322939, 0.7807638039626498, 0.6845730741306342, 0.46020231128256656, 0.9301610108102609, 0.7362785167282615, 0.6928492942953581, 0.6895477163568915, 1.2669417533620937, 1.1062817563047462, 1.3390348825523504, 1.3317342139895063, 2.144680150249246, 1.4856156034891146, 1.1506534528333585, 0.316248434124413, 0.42389451490188673, 0.302561592757571, 0.3760595766394344, 0.27322836970775194, 0.21350954899494423, 0.2911362329789658, 0.3512495348226712 ], "peak_layer": 1, "peak_normalized": 3.297293025983657, "population_cv": 0.6538528745243442, "rest_mean": 0.013146174588689098, "spike_contrast": 1.3576371474988167, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.017847734969109297, "top_five_layers": [ 1, 22, 2, 8, 23 ], "top_five_spike_overlap": 2 }, "values": [ 0.045769043266773224, 0.02782859094440937, 0.019777558743953705, 0.017386535182595253, 0.007129390724003315, 0.015810219570994377, 0.02059624344110489, 0.021884648129343987, 0.013784985989332199, 0.013425199314951897, 0.010837621055543423, 0.009502417407929897, 0.006387973204255104, 0.012911372818052769, 0.01022012997418642, 0.009617297910153866, 0.009571469388902187, 0.017586156725883484, 0.01535606849938631, 0.018586866557598114, 0.018485527485609055, 0.029769862070679665, 0.02062152326107025, 0.01597198285162449, 0.0043897791765630245, 0.005883992183953524, 0.004199794959276915, 0.005220005288720131, 0.0037926265504211187, 0.0029636819381266832, 0.004041201900690794, 0.004875622224062681 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 2.3228319712131906e-06, "first_to_last_ratio": 2.4976448026540363, "imbalance_abs_log_ratio": 0.915348208900507, "last_quartile_mean": 9.300089303110326e-07, "mean": 1.3029091805449866e-06, "normalized": [ 9.073230145097602, 2.3402488351091066, 0.44311573780024827, 0.533415774033705, 0.3017962240685419, 0.4548687484313705, 0.48149182110792027, 0.634266387535609, 0.5450840153159764, 0.5908968965833815, 0.5042485738178415, 0.5092960366467646, 0.3250322093633585, 0.6385171580951731, 0.5646127267899368, 0.6647842256833941, 0.42505519829104743, 0.7928879885145004, 0.7062795418090991, 0.8682122955059692, 0.8336575516200646, 1.511249627521053, 1.3630971501954239, 1.1843020584287967, 0.5134216818885189, 0.7180137487400903, 0.7642083756598086, 1.0412903643835534, 0.6569051263038723, 0.6942292931060672, 0.657983655934825, 0.6643008266173792 ], "peak_layer": 1, "peak_normalized": 9.073230145097602, "population_cv": 1.501914025430085, "rest_mean": 1.2833304091836233e-06, "spike_contrast": 1.0976398095268665, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.4086345458963478e-06, "top_five_layers": [ 1, 2, 22, 23, 24 ], "top_five_spike_overlap": 3 }, "values": [ 1.1821594853245188e-05, 3.0491316920233658e-06, 5.773395628239086e-07, 6.949923090360244e-07, 3.932130709927151e-07, 5.926526682742406e-07, 6.273401140788337e-07, 8.263914992312493e-07, 7.101949677235098e-07, 7.698849913140293e-07, 6.569900961039821e-07, 6.635664817622455e-07, 4.2348744955233997e-07, 8.319298672176956e-07, 7.356391051871469e-07, 8.661534707243845e-07, 5.538083200917754e-07, 1.0330610393793904e-06, 9.20218099054182e-07, 1.131201770476764e-06, 1.0861800774364383e-06, 1.9690210137923714e-06, 1.7759917909643264e-06, 1.5430380244652042e-06, 6.68941822823399e-07, 9.355067049909849e-07, 9.956941084965365e-07, 1.356706775368366e-06, 8.558877198083792e-07, 9.045177193911513e-07, 8.572929459660372e-07, 8.655236456434068e-07 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 1.518431281510857e-05, "first_to_last_ratio": 4.663264423196815, "imbalance_abs_log_ratio": 1.5397157228872644, "last_quartile_mean": 3.2561552245624625e-06, "mean": 1.0378024420276688e-05, "normalized": [ 1.8677548873043361, 2.0698082385573993, 1.4922469847228352, 1.2997577654050716, 0.6036518944688829, 1.2750551131299375, 1.6243384663630878, 1.4723603137768235, 1.0912552372766198, 1.0337226903691044, 0.8377350182606691, 0.7325625183313245, 0.5189047676224301, 1.0059234867185796, 0.8043726436895072, 0.7238036352303707, 0.7298742658425005, 1.3026265492753781, 1.176983543417771, 1.385767297373008, 1.3279997091231934, 2.2747216976580664, 1.6420468421546672, 1.196687842613848, 0.33569562813305825, 0.4516974037627159, 0.3117650323326891, 0.33395347839260336, 0.2896404514398891, 0.23141270534962813, 0.2501638908699, 0.30571000103410495 ], "peak_layer": 22, "peak_normalized": 2.2747216976580664, "population_cv": 0.5549670795354054, "rest_mean": 9.694938348224645e-06, "spike_contrast": 1.4509312699171146, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.4066689209357719e-05, "top_five_layers": [ 22, 2, 1, 23, 7 ], "top_five_spike_overlap": 2 }, "values": [ 1.9383605831535533e-05, 2.1480520445038565e-05, 1.5486575648537837e-05, 1.3488917829818092e-05, 6.264714102144353e-06, 1.3232553101261146e-05, 1.6857424270710908e-05, 1.528019129182212e-05, 1.1325073501211591e-05, 1.0727999324444681e-05, 8.694034477230161e-06, 7.602551704621874e-06, 5.38520635018358e-06, 1.043949851009529e-05, 8.347798939212225e-06, 7.511651801905828e-06, 7.574652954644989e-06, 1.3518690138880629e-05, 1.2214763955853414e-05, 1.4381526852957904e-05, 1.378201341140084e-05, 2.3607117327628657e-05, 1.7041202227119356e-05, 1.241925565409474e-05, 3.4838574265450006e-06, 4.6877266868250445e-06, 3.2355051189369988e-06, 3.465777353994781e-06, 3.0058956781431334e-06, 2.4016067072807346e-06, 2.5962069685192546e-06, 3.1726658562547527e-06 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 9.527698239253368e-06, "first_to_last_ratio": 4.389907779560697, "imbalance_abs_log_ratio": 1.4793082199326493, "last_quartile_mean": 2.1703640982195793e-06, "mean": 6.909023269940917e-06, "normalized": [ 1.129596250120822, 2.0359943796947944, 1.4980953024591086, 1.2898005234040295, 0.6933396099752838, 1.3571829409739298, 1.7273711808510577, 1.3007994616537462, 1.148003549990779, 1.1031718913753568, 0.893231497184441, 0.7589494469027065, 0.5765234455451127, 1.0576809204322937, 0.8384613815235499, 0.7353344550416073, 0.7517648052187398, 1.3176053769842697, 1.2428665950726396, 1.3898625065048276, 1.272970254917748, 2.3499921225562854, 1.8037824321891982, 1.2145419003459774, 0.34311480371137404, 0.4798252914478828, 0.32471069607940173, 0.30212380230732616, 0.30596350513827214, 0.2506034178243104, 0.23473572427419578, 0.2720005282989316 ], "peak_layer": 22, "peak_normalized": 2.3499921225562854, "population_cv": 0.5422973890411167, "rest_mean": 6.401235261229751e-06, "spike_contrast": 1.5076900196802223, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.651078516981216e-06, "top_five_layers": [ 22, 2, 23, 7, 3 ], "top_five_spike_overlap": 2 }, "values": [ 7.804406777722761e-06, 1.4066732546780258e-05, 1.0350375305279158e-05, 8.911261829780415e-06, 4.790299499290995e-06, 9.37680852075573e-06, 1.1934447684325278e-05, 8.98725375009235e-06, 7.931583240861073e-06, 7.621840268257074e-06, 6.171357199491467e-06, 5.243599389359588e-06, 3.983213900937699e-06, 7.307542091439245e-06, 5.792949195893016e-06, 5.080442861071788e-06, 5.193960532778874e-06, 9.103366210183594e-06, 8.586994226789102e-06, 9.602592399460264e-06, 8.794981113169342e-06, 1.6236150258919224e-05, 1.2462374797905795e-05, 8.391298251808621e-06, 2.3705881631030934e-06, 3.3151241041196045e-06, 2.2434337552112993e-06, 2.087380380544346e-06, 2.1139089767530095e-06, 1.731424845274887e-06, 1.6217945812968537e-06, 1.8792579794535413e-06 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 4.661776938519324e-05, "first_to_last_ratio": 4.956483491490523, "imbalance_abs_log_ratio": 1.6006965158064674, "last_quartile_mean": 9.405412015439651e-06, "mean": 3.052339087616929e-05, "normalized": [ 2.269640871871418, 2.103413483203614, 1.5010930965115201, 1.3600293337560434, 0.5417036375547336, 1.2186373311377228, 1.54879395268707, 1.6749295953186458, 1.0853724235693705, 0.9435662475583906, 0.7739402533390257, 0.7282021022983288, 0.4698819487203021, 0.9465395953646257, 0.7689807206401379, 0.7138816549924055, 0.7055456854472902, 1.2366945759381338, 1.1159292425803522, 1.4065273588873586, 1.3669593187179923, 2.281168360271765, 1.5404756823502455, 1.2329907335351769, 0.3299485217670214, 0.44398198806845046, 0.30141128968992753, 0.34190180425233946, 0.2717856158962549, 0.2109529304584195, 0.2507572580381302, 0.3143633855777871 ], "peak_layer": 22, "peak_normalized": 2.281168360271765, "population_cv": 0.5886110353897875, "rest_mean": 2.8543279027130835e-05, "spike_contrast": 1.44398248084253, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.1215994860976936e-05, "top_five_layers": [ 22, 1, 2, 8, 7 ], "top_five_spike_overlap": 1 }, "values": [ 6.927713548066095e-05, 6.420331192202866e-05, 4.581845132634044e-05, 4.151270695729181e-05, 1.653463186812587e-05, 3.7196943594608456e-05, 4.727444320451468e-05, 5.112453072797507e-05, 3.312924673082307e-05, 2.880084139178507e-05, 2.3623280867468566e-05, 2.2227197405300103e-05, 1.4342390386445913e-05, 2.8891598049085587e-05, 2.347189911233727e-05, 2.1790088794659823e-05, 2.1535646737902425e-05, 3.774811193579808e-05, 3.406194446142763e-05, 4.293198435334489e-05, 4.172423359705135e-05, 6.962899351492524e-05, 4.702054138761014e-05, 3.76350581063889e-05, 1.0071147698909044e-05, 1.3551835763792042e-05, 9.200094609695952e-06, 1.0436002412461676e-05, 8.295818588521797e-06, 6.438998752855696e-06, 7.65396180213429e-06, 9.595436495146714e-06 ] } } }, "pre_mlp_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "e53fa425d89dfeb0e9395eca9bcd73dc0a55314abf45c5b10b43e2cbfa724345", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.059514161199331284, 0.08157797157764435, 0.14292451739311218, 0.10997853428125381, 0.0991024449467659, 0.1065012738108635, 0.12155740708112717, 0.1095810979604721, 0.06964563578367233, 0.08311934769153595, 0.09363401681184769, 0.0941167026758194, 0.08564268052577972, 0.09493856877088547, 0.09063401818275452, 0.09473779052495956, 0.10505300015211105, 0.09355910867452621, 0.09618847072124481, 0.11416846513748169, 0.047205954790115356, 0.053352247923612595, 0.06512638926506042, 0.07829347252845764, 0.05368548259139061, 0.05310077592730522, 0.061235878616571426, 0.07320034503936768, 0.05304349213838577, 0.0639304369688034, 0.07404901832342148, 0.08157160878181458 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 2.5659798126298483e-06, "first_to_last_ratio": 0.36599966263085826, "imbalance_abs_log_ratio": 1.005122867354807, "last_quartile_mean": 7.0108802674440085e-06, "mean": 4.2533971615910104e-06, "normalized": [ 1.4761153336451707, 0.514656386601766, 0.3457474192273021, 0.5970123906555977, 0.3476618465526744, 0.28228406652365334, 0.3782865629368856, 0.8844578868908509, 0.8374507777409872, 0.6279543473598765, 0.5822744141180622, 0.6000452711946552, 0.5313635036616726, 0.564111236423224, 0.6699814597152032, 0.7225405551152823, 0.49009606814322876, 0.6391164810311278, 0.6076017864737518, 0.5380966748663432, 1.6069612894199934, 1.8258207060779335, 1.642064977255438, 1.50188778863712, 1.3807564590119727, 1.7630953801429854, 2.110541233854943, 1.5785396144799588, 1.942924160795935, 1.8970854603874512, 1.2721833449856, 1.2412851160733533 ], "peak_layer": 27, "peak_normalized": 2.110541233854943, "population_cv": 0.5612932461404013, "rest_mean": 3.787494022246931e-06, "spike_contrast": 1.787269913638878, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.7692741140490394e-06, "top_five_layers": [ 27, 29, 30, 22, 26 ], "top_five_spike_overlap": 1 }, "values": [ 6.278504770307336e-06, 2.1890380139666377e-06, 1.470601091568824e-06, 2.5393308078491827e-06, 1.4787439113206347e-06, 1.200666247314075e-06, 1.6090029930637684e-06, 3.761950665648328e-06, 3.5620107610156992e-06, 2.6709392386692343e-06, 2.4766443402768346e-06, 2.5522308533254545e-06, 2.2601000182476128e-06, 2.3993891318241367e-06, 2.849697239071247e-06, 3.0732519462617347e-06, 2.0845732251473237e-06, 2.718416226343834e-06, 2.5843717139650835e-06, 2.288738869538065e-06, 6.83504458720563e-06, 7.765940608805977e-06, 6.984354513406288e-06, 6.388125257217325e-06, 5.872905603609979e-06, 7.499144885514397e-06, 8.976970093499403e-06, 6.714155915688025e-06, 8.264028110716026e-06, 8.06905791250756e-06, 5.411101028585108e-06, 5.279678589431569e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 1.0336362720408943e-05, "first_to_last_ratio": 0.36916845721449065, "imbalance_abs_log_ratio": 0.9965022155003058, "last_quartile_mean": 2.7999040867143776e-05, "mean": 1.7000353636831278e-05, "normalized": [ 1.5156669641989942, 0.5287102646352508, 0.3463843108151677, 0.5984814482856154, 0.3500056561368898, 0.2847027805055364, 0.3782587085716728, 0.8618593756240107, 0.8326173752726855, 0.6278674133034854, 0.575990647165045, 0.6049732864225724, 0.528894032241933, 0.5581362465875599, 0.6699313102365538, 0.715047354397809, 0.49083166825704555, 0.6414114336566974, 0.6129387428203908, 0.5273538082546323, 1.6115310615537757, 1.818889935053532, 1.6473560616146303, 1.4964149668363669, 1.3763202946872732, 1.7646639929683612, 2.1192887161098897, 1.5805908024232607, 1.941142739736412, 1.886880418619933, 1.2749367493029737, 1.2319214337040436 ], "peak_layer": 27, "peak_normalized": 2.1192887161098897, "population_cv": 0.5622195166215849, "rest_mean": 1.5142585012923788e-05, "spike_contrast": 1.785184245811424, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.7032304205931724e-05, "top_five_layers": [ 27, 29, 30, 22, 26 ], "top_five_spike_overlap": 1 }, "values": [ 2.576687438704539e-05, 8.988261470221914e-06, 5.888655778107932e-06, 1.0174396265938412e-05, 5.950219929218292e-06, 4.840047949983273e-06, 6.430531811929541e-06, 1.4651914170826785e-05, 1.4154789823805913e-05, 1.0673968063201755e-05, 9.792044693313073e-06, 1.028475981001975e-05, 8.991385584522504e-06, 9.488513569522183e-06, 1.1389069186407141e-05, 1.2156057891843375e-05, 8.344311936525628e-06, 1.09042211988708e-05, 1.0420175385661423e-05, 8.965201232058462e-06, 2.73965979431523e-05, 3.092177212238312e-05, 2.8005635613226332e-05, 2.5439583623665385e-05, 2.339793172723148e-05, 2.9999911930644885e-05, 3.602865763241425e-05, 2.687060259631835e-05, 3.300011303508654e-05, 3.2077634386951104e-05, 2.1674375602742657e-05, 2.094310002576094e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.009166389761958271, "first_to_last_ratio": 0.369168456003872, "imbalance_abs_log_ratio": 0.9965022187796181, "last_quartile_mean": 0.024829829344525933, "mean": 0.015076083509484306, "normalized": [ 1.515666956303773, 0.5287102436585791, 0.34638431937697917, 0.5984814606528517, 0.3500056528800389, 0.28470276415457657, 0.37825871353114154, 0.8618593972511569, 0.8326173523755421, 0.6278673791197023, 0.5759906639691527, 0.6049733163116636, 0.5288940237933693, 0.5581362539934898, 0.669931285244655, 0.7150473624283348, 0.49083167528227745, 0.6414114499676391, 0.612938748889899, 0.5273537918267395, 1.611530998848797, 1.81888997049547, 1.6473560790400137, 1.4964149524560115, 1.376320224993216, 1.7646640201362025, 2.119288762770415, 1.580590825527721, 1.9411428636016368, 1.886880342539917, 1.2749367244706349, 1.2319214241084036 ], "peak_layer": 27, "peak_normalized": 2.119288762770415, "population_cv": 0.5622195220870171, "rest_mean": 0.013428595779394661, "spike_contrast": 1.785184217753632, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.023972517251968382, "top_five_layers": [ 27, 29, 30, 22, 26 ], "top_five_spike_overlap": 1 }, "values": [ 0.022850321605801582, 0.007970879785716534, 0.005222118925303221, 0.009022756479680538, 0.005276714451611042, 0.004292202647775412, 0.0057026599533855915, 0.01299346424639225, 0.012552608735859394, 0.009465781040489674, 0.008683683350682259, 0.009120628237724304, 0.007973650470376015, 0.008414508774876595, 0.010099940001964569, 0.010780113749206066, 0.007399819325655699, 0.009669972583651543, 0.009240715764462948, 0.007950429804623127, 0.02429557591676712, 0.02742173708975315, 0.024835677817463875, 0.022560076788067818, 0.020749518647789955, 0.026604222133755684, 0.03195057436823845, 0.02382911927998066, 0.02926483191549778, 0.02844676561653614, 0.019221052527427673, 0.018572550266981125 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 6.295395351685329e-07, "first_to_last_ratio": 0.3328410918755602, "imbalance_abs_log_ratio": 1.100090104473495, "last_quartile_mean": 1.8914116992618801e-06, "mean": 1.1039766860676536e-06, "normalized": [ 1.7691588153576192, 0.4723976315102391, 0.331620773325477, 0.6084747938587679, 0.31459851305752184, 0.23863723881483964, 0.2880555256593535, 0.5390337364705848, 0.7117363401356493, 0.5533012725046477, 0.5306832413013026, 0.5502501993519945, 0.5063769660968573, 0.5565551120619713, 0.6671832884851685, 0.6985988998480835, 0.4936209099533731, 0.6297408087457507, 0.633376856282218, 0.486423165994797, 1.5856447152338804, 1.8569645979923681, 1.7267192610693793, 1.5446760154697174, 1.3867413761492773, 1.741750440716359, 2.2661377285139856, 1.651104898622397, 1.9775721974614733, 1.9755912860545461, 1.3719953464131824, 1.3352780474872175 ], "peak_layer": 27, "peak_normalized": 2.2661377285139856, "population_cv": 0.6165617849443572, "rest_mean": 9.77193306223651e-07, "spike_contrast": 1.8303511964662476, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.7886069372252678e-06, "top_five_layers": [ 27, 29, 30, 22, 1 ], "top_five_spike_overlap": 1 }, "values": [ 1.9531100861058803e-06, 5.215159717408824e-07, 3.661016023670527e-07, 6.717419864799012e-07, 3.4730942388705444e-07, 2.634499480791419e-07, 3.1800658462088904e-07, 5.950806780674611e-07, 7.857403261368745e-07, 6.108317052166967e-07, 5.85861926083453e-07, 6.074633915886807e-07, 5.590283649326011e-07, 6.144238682281866e-07, 7.365547958215757e-07, 7.712368983447959e-07, 5.449459763440245e-07, 6.952191711206979e-07, 6.992332828303915e-07, 5.369998348214722e-07, 1.7505147980045876e-06, 2.050045623036567e-06, 1.906257807604561e-06, 1.7052863086064463e-06, 1.5309301488741767e-06, 1.9228518794989213e-06, 2.50176321969775e-06, 1.8227813143312233e-06, 2.183193601013045e-06, 2.181006721002632e-06, 1.5146508758334676e-06, 1.4741158338438254e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 6.89456643954145e-06, "first_to_last_ratio": 0.3796792403120553, "imbalance_abs_log_ratio": 0.9684284871604251, "last_quartile_mean": 1.8158923921873793e-05, "mean": 1.1488302867235234e-05, "normalized": [ 1.5136566371884412, 0.5333513797853834, 0.2675347114339333, 0.5361211171438036, 0.37674844561478477, 0.3041039964791309, 0.3915881847455752, 0.8779990634182819, 0.8889302080257989, 0.6659672215101894, 0.6117341711918809, 0.6508368602493252, 0.570205250475387, 0.5980660643721846, 0.7074073564138591, 0.7516634622155739, 0.5108412668464613, 0.6718790978886663, 0.6307907812214575, 0.5604325172866399, 1.6813360270315938, 1.901065357452136, 1.6991696649376673, 1.453414335057172, 1.3855556906088564, 1.7229017208288337, 2.0042410428165356, 1.5072214668083879, 1.8982072696995007, 1.7867537223997227, 1.2048957595138885, 1.135380149338947 ], "peak_layer": 27, "peak_normalized": 2.0042410428165356, "population_cv": 0.5434642371113595, "rest_mean": 1.016053542352219e-05, "spike_contrast": 1.8363448662450228, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.8658247063285673e-05, "top_five_layers": [ 27, 22, 29, 30, 26 ], "top_five_spike_overlap": 1 }, "values": [ 1.7389345885021612e-05, 6.1273021856322885e-06, 3.073519792451407e-06, 6.159121767268516e-06, 4.32820024798275e-06, 3.493638814688893e-06, 4.4986836655880325e-06, 1.0086719157698099e-05, 1.02122994576348e-05, 7.650833140360191e-06, 7.027787432889454e-06, 7.4770109677047e-06, 6.550690613948973e-06, 6.87076408212306e-06, 8.126909960992634e-06, 8.63533750816714e-06, 5.868699190614279e-06, 7.718750566709787e-06, 7.246715540532023e-06, 6.438418495235965e-06, 1.9315697500132956e-05, 2.1840014596818946e-05, 1.9520575733622536e-05, 1.66972640727181e-05, 1.591768341313582e-05, 1.979321677936241e-05, 2.3025328118819743e-05, 1.7315416698693298e-05, 2.180718001909554e-05, 2.0526767912087962e-05, 1.384220740874298e-05, 1.3043591025052592e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 4.4220539336947695e-06, "first_to_last_ratio": 0.36296705738388124, "imbalance_abs_log_ratio": 1.0134431998439999, "last_quartile_mean": 1.218307238559646e-05, "mean": 7.758772476051945e-06, "normalized": [ 1.418774568217446, 0.5159243867351231, 0.21221206095148934, 0.44763584816794444, 0.38210324493100417, 0.3074227076140585, 0.39146537193894304, 0.8840015554885894, 0.8997064189242463, 0.64738955285881, 0.5886483512194688, 0.6533612833393894, 0.5687733340493856, 0.6148004607950276, 0.712501272413854, 0.7597788005617484, 0.52088877452795, 0.693309246128764, 0.6395924496740145, 0.5859673790182021, 1.7636633377781363, 1.9743419660921866, 1.7995720246024307, 1.4563100545786727, 1.44908663325969, 1.771082518949164, 1.9370702280346128, 1.5139633237708119, 1.851341479415208, 1.791371327902312, 1.1888872095590746, 1.0590528285022418 ], "peak_layer": 22, "peak_normalized": 1.9743419660921866, "population_cv": 0.553760145638796, "rest_mean": 6.769392771106881e-06, "spike_contrast": 1.9353911533505301, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.3101422882755288e-05, "top_five_layers": [ 22, 27, 29, 23, 30 ], "top_five_spike_overlap": 2 }, "values": [ 1.1007949069608003e-05, 4.002939931524452e-06, 1.6465050975966733e-06, 3.4731046980596147e-06, 2.96465213978081e-06, 2.3852228423493216e-06, 3.0372907531273086e-06, 6.858766937511973e-06, 6.980617399676703e-06, 5.02294824400451e-06, 4.567188625514973e-06, 5.06928154209163e-06, 4.412982889334671e-06, 4.770096893480513e-06, 5.528135261556599e-06, 5.894950845686253e-06, 4.041457486891886e-06, 5.3792286962561775e-06, 4.962452294421382e-06, 4.546387572190724e-06, 1.3683862562174909e-05, 1.531847010483034e-05, 1.3962469893158413e-05, 1.1299178368062712e-05, 1.1243133485550061e-05, 1.3741426300839521e-05, 1.502928716945462e-05, 1.1746496966225095e-05, 1.4364137314260006e-05, 1.3898842553317081e-05, 9.224305358657148e-06, 8.216949936468154e-06 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 2.1320740415831096e-05, "first_to_last_ratio": 0.34832920155792996, "imbalance_abs_log_ratio": 1.0546072648778662, "last_quartile_mean": 6.120859325164929e-05, "mean": 3.639655466258773e-05, "normalized": [ 1.5098932256004431, 0.5178228475860206, 0.2840240499721815, 0.5533689652135182, 0.35174191539514554, 0.2755395852631113, 0.3811312955186602, 0.8127987001290801, 0.8329172779480924, 0.6385514066227361, 0.5850044351668731, 0.6044354298768252, 0.5319071564581624, 0.5541528539375777, 0.6818582475147731, 0.7197712742893193, 0.4904860464769455, 0.6496495917293342, 0.6235624646747766, 0.5212841023748116, 1.5068509277242306, 1.7559963024603593, 1.6377441021279566, 1.5257962962479101, 1.3573209425897699, 1.7786138785496393, 2.216847506941414, 1.6356616618758617, 1.9934934455934286, 1.9269447163430042, 1.2842333269588233, 1.2605960208392144 ], "peak_layer": 27, "peak_normalized": 2.216847506941414, "population_cv": 0.5736692714518858, "rest_mean": 3.264405832278835e-05, "spike_contrast": 1.7356921231191043, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.666003489750438e-05, "top_five_layers": [ 27, 29, 30, 26, 22 ], "top_five_spike_overlap": 1 }, "values": [ 5.495491132023744e-05, 1.8846967577701434e-05, 1.0337496860302053e-05, 2.0140723790973425e-05, 1.2802193850802723e-05, 1.0028691576735582e-05, 1.3871866030967794e-05, 2.9583072318928316e-05, 3.0315319236251526e-05, 2.3241071176016703e-05, 2.1292145902407356e-05, 2.199936716351658e-05, 1.9359587895451114e-05, 2.016925463976804e-05, 2.4817290977807716e-05, 2.6197194529231638e-05, 1.7852002201834694e-05, 2.3645006876904517e-05, 2.269552533107344e-05, 1.897294532682281e-05, 5.484418215928599e-05, 6.391221540980041e-05, 5.960824273643084e-05, 5.553372830036096e-05, 4.940180588164367e-05, 6.473541725426912e-05, 8.068561146501452e-05, 5.953244908596389e-05, 7.255629316205159e-05, 7.013414870016277e-05, 4.6741668484173715e-05, 4.588135197991505e-05 ] } } } }, "selector": { "expected_uniform_count": 57, "identities_unique": true, "identity_order_sha256": "97fd9721a263024f6a746638f32e0ad9a988cd4c111c00e9e178c1022e5c9d5e", "passed": true, "uniform_count": 57, "uniform_indices": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64 ], "visit_count": 65, "visits": [ { "branch": "attention", "coefficient": "uniform", "group": 1, "index": 0, "kind": "depth", "layer": 1, "offset": 1, "sources": 1, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 1, "index": 1, "kind": "depth", "layer": 1, "offset": 1, "sources": 2, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 1, "index": 2, "kind": "depth", "layer": 2, "offset": 2, "sources": 2, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 1, "index": 3, "kind": "depth", "layer": 2, "offset": 2, "sources": 2, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 1, "index": 4, "kind": "depth", "layer": 3, "offset": 3, "sources": 2, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 1, "index": 5, "kind": "depth", "layer": 3, "offset": 3, "sources": 2, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 1, "index": 6, "kind": "depth", "layer": 4, "offset": 4, "sources": 2, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 1, "index": 7, "kind": "depth", "layer": 4, "offset": 4, "sources": 2, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 2, "index": 8, "kind": "depth", "layer": 5, "offset": 1, "sources": 2, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 2, "index": 9, "kind": "depth", "layer": 5, "offset": 1, "sources": 3, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 2, "index": 10, "kind": "depth", "layer": 6, "offset": 2, "sources": 3, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 2, "index": 11, "kind": "depth", "layer": 6, "offset": 2, "sources": 3, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 2, "index": 12, "kind": "depth", "layer": 7, "offset": 3, "sources": 3, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 2, "index": 13, "kind": "depth", "layer": 7, "offset": 3, "sources": 3, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 2, "index": 14, "kind": "depth", "layer": 8, "offset": 4, "sources": 3, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 2, "index": 15, "kind": "depth", "layer": 8, "offset": 4, "sources": 3, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 3, "index": 16, "kind": "depth", "layer": 9, "offset": 1, "sources": 3, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 3, "index": 17, "kind": "depth", "layer": 9, "offset": 1, "sources": 4, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 3, "index": 18, "kind": "depth", "layer": 10, "offset": 2, "sources": 4, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 3, "index": 19, "kind": "depth", "layer": 10, "offset": 2, "sources": 4, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 3, "index": 20, "kind": "depth", "layer": 11, "offset": 3, "sources": 4, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 3, "index": 21, "kind": "depth", "layer": 11, "offset": 3, "sources": 4, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 3, "index": 22, "kind": "depth", "layer": 12, "offset": 4, "sources": 4, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 3, "index": 23, "kind": "depth", "layer": 12, "offset": 4, "sources": 4, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 4, "index": 24, "kind": "depth", "layer": 13, "offset": 1, "sources": 4, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 4, "index": 25, "kind": "depth", "layer": 13, "offset": 1, "sources": 5, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 4, "index": 26, "kind": "depth", "layer": 14, "offset": 2, "sources": 5, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 4, "index": 27, "kind": "depth", "layer": 14, "offset": 2, "sources": 5, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 4, "index": 28, "kind": "depth", "layer": 15, "offset": 3, "sources": 5, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 4, "index": 29, "kind": "depth", "layer": 15, "offset": 3, "sources": 5, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 4, "index": 30, "kind": "depth", "layer": 16, "offset": 4, "sources": 5, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 4, "index": 31, "kind": "depth", "layer": 16, "offset": 4, "sources": 5, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 5, "index": 32, "kind": "depth", "layer": 17, "offset": 1, "sources": 5, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 5, "index": 33, "kind": "depth", "layer": 17, "offset": 1, "sources": 6, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 5, "index": 34, "kind": "depth", "layer": 18, "offset": 2, "sources": 6, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 5, "index": 35, "kind": "depth", "layer": 18, "offset": 2, "sources": 6, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 5, "index": 36, "kind": "depth", "layer": 19, "offset": 3, "sources": 6, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 5, "index": 37, "kind": "depth", "layer": 19, "offset": 3, "sources": 6, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 5, "index": 38, "kind": "depth", "layer": 20, "offset": 4, "sources": 6, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 5, "index": 39, "kind": "depth", "layer": 20, "offset": 4, "sources": 6, "use_uniform": true }, { "branch": "attention", "coefficient": "detached_learned", "group": 6, "index": 40, "kind": "depth", "layer": 21, "offset": 1, "sources": 6, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 6, "index": 41, "kind": "depth", "layer": 21, "offset": 1, "sources": 7, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 6, "index": 42, "kind": "depth", "layer": 22, "offset": 2, "sources": 7, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 6, "index": 43, "kind": "depth", "layer": 22, "offset": 2, "sources": 7, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 6, "index": 44, "kind": "depth", "layer": 23, "offset": 3, "sources": 7, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 6, "index": 45, "kind": "depth", "layer": 23, "offset": 3, "sources": 7, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 6, "index": 46, "kind": "depth", "layer": 24, "offset": 4, "sources": 7, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 6, "index": 47, "kind": "depth", "layer": 24, "offset": 4, "sources": 7, "use_uniform": false }, { "branch": "attention", "coefficient": "uniform", "group": 7, "index": 48, "kind": "depth", "layer": 25, "offset": 1, "sources": 7, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 7, "index": 49, "kind": "depth", "layer": 25, "offset": 1, "sources": 8, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 7, "index": 50, "kind": "depth", "layer": 26, "offset": 2, "sources": 8, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 7, "index": 51, "kind": "depth", "layer": 26, "offset": 2, "sources": 8, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 7, "index": 52, "kind": "depth", "layer": 27, "offset": 3, "sources": 8, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 7, "index": 53, "kind": "depth", "layer": 27, "offset": 3, "sources": 8, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 7, "index": 54, "kind": "depth", "layer": 28, "offset": 4, "sources": 8, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 7, "index": 55, "kind": "depth", "layer": 28, "offset": 4, "sources": 8, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 8, "index": 56, "kind": "depth", "layer": 29, "offset": 1, "sources": 8, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 8, "index": 57, "kind": "depth", "layer": 29, "offset": 1, "sources": 9, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 8, "index": 58, "kind": "depth", "layer": 30, "offset": 2, "sources": 9, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 8, "index": 59, "kind": "depth", "layer": 30, "offset": 2, "sources": 9, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 8, "index": 60, "kind": "depth", "layer": 31, "offset": 3, "sources": 9, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 8, "index": 61, "kind": "depth", "layer": 31, "offset": 3, "sources": 9, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 8, "index": 62, "kind": "depth", "layer": 32, "offset": 4, "sources": 9, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 8, "index": 63, "kind": "depth", "layer": 32, "offset": 4, "sources": 9, "use_uniform": true }, { "branch": "output", "coefficient": "uniform", "group": null, "index": 64, "kind": "output", "layer": null, "offset": null, "sources": 9, "use_uniform": true } ], "visits_sha256": "9b343b963f66443eda485b5123dbb230016e3c1d527af25d484ae56d6fa0c934" } }, "uniform_except_group_7": { "bits_per_byte": 1.8740129107275643, "forward": { "activation_sha256": { "attention_branch_output": "5e6ecbcd78543fc42d4098a51169dca446d404ce384015643e84d2c78433c949", "mlp_branch_output": "c0b8018c349b1bd09e5d17858eb312a2323da0e92de9651e2bb08ae4adf115c7", "post_attention_state": "90a633f12d984935028fe5d7212be48e7da7b4a0806751939a551938d16dc540", "post_mlp_state": "89f2226456bfe96e245d2aad0f699d32c676f9f53099789cd870253d002402f5", "pre_attention_input": "ee695e78937de01b7128d3866b8fc5dd328ebdf2c159be37918d4f02952d5248", "pre_mlp_input": "e53fa425d89dfeb0e9395eca9bcd73dc0a55314abf45c5b10b43e2cbfa724345" }, "logits_sha256": "bdacba5675f0579cec629bbf01b36c8660ad576cfdf9fef1bea8259eaeac9ec2", "loss_nats": 1.2989667654037476, "mixer_summary_sha256": "eb3ac0f07de9b755d6e67a510ab68d09d9ad666bb7b5d020ba7d8c5055c399b2" }, "loss_nats": 1.2989667654037476, "loss_scale": 1.0, "mixers": [ { "branch": "attention", "entropy_mean": -0.0, "group": 1, "latest_source_mass": 1.0, "layer": 1, "max_source_mass": 1.0, "mixer_index": 0, "normalized_entropy": 1.0, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 1.0, "median": 1.0, "p05": 1.0, "p95": 1.0 } ], "sources": 1 }, { "branch": "mlp", "entropy_mean": 0.22886337339878082, "group": 1, "latest_source_mass": 0.06350618600845337, "layer": 1, "max_source_mass": 0.9364938139915466, "mixer_index": 1, "normalized_entropy": 0.3301800538435402, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.9364938139915466, "median": 0.9452826976776123, "p05": 0.8725647330284119, "p95": 0.9712384939193726 }, { "label": "current_group_partial", "mean": 0.06350618600845337, "median": 0.0547172985970974, "p05": 0.028761589899659157, "p95": 0.12743529677391052 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.603729784488678, "group": 1, "latest_source_mass": 0.4133155345916748, "layer": 2, "max_source_mass": 0.5866844654083252, "mixer_index": 2, "normalized_entropy": 0.8709979661187783, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.5866844654083252, "median": 0.5734395384788513, "p05": 0.29813364148139954, "p95": 0.8843642473220825 }, { "label": "current_group_partial", "mean": 0.4133155345916748, "median": 0.4265604615211487, "p05": 0.11563576757907867, "p95": 0.7018663287162781 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.3967982828617096, "group": 1, "latest_source_mass": 0.14110636711120605, "layer": 2, "max_source_mass": 0.858893632888794, "mixer_index": 3, "normalized_entropy": 0.5724589149178446, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.858893632888794, "median": 0.8705973029136658, "p05": 0.7599387168884277, "p95": 0.9270997643470764 }, { "label": "current_group_partial", "mean": 0.14110636711120605, "median": 0.12940272688865662, "p05": 0.07290027290582657, "p95": 0.2400612235069275 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6852496862411499, "group": 1, "latest_source_mass": 0.518975019454956, "layer": 3, "max_source_mass": 0.518975019454956, "mixer_index": 4, "normalized_entropy": 0.9886063241108252, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.48102498054504395, "median": 0.48102718591690063, "p05": 0.38786420226097107, "p95": 0.580143928527832 }, { "label": "current_group_partial", "mean": 0.518975019454956, "median": 0.5189728140830994, "p05": 0.41985607147216797, "p95": 0.6121357679367065 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.5890734195709229, "group": 1, "latest_source_mass": 0.3123423159122467, "layer": 3, "max_source_mass": 0.6876576542854309, "mixer_index": 5, "normalized_entropy": 0.8498533011344741, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.6876576542854309, "median": 0.6909762620925903, "p05": 0.48803940415382385, "p95": 0.8900076746940613 }, { "label": "current_group_partial", "mean": 0.3123423159122467, "median": 0.3090237081050873, "p05": 0.10999235510826111, "p95": 0.5119606256484985 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6873323321342468, "group": 1, "latest_source_mass": 0.4831485152244568, "layer": 4, "max_source_mass": 0.5168514251708984, "mixer_index": 6, "normalized_entropy": 0.9916109470127238, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.5168514251708984, "median": 0.5219584703445435, "p05": 0.4251725673675537, "p95": 0.5947866439819336 }, { "label": "current_group_partial", "mean": 0.4831485152244568, "median": 0.47804147005081177, "p05": 0.405213326215744, "p95": 0.5748274326324463 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.487301766872406, "group": 1, "latest_source_mass": 0.2060384750366211, "layer": 4, "max_source_mass": 0.7939615249633789, "mixer_index": 7, "normalized_entropy": 0.7030278424832499, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.7939615249633789, "median": 0.7953404188156128, "p05": 0.6590759754180908, "p95": 0.9329437017440796 }, { "label": "current_group_partial", "mean": 0.2060384750366211, "median": 0.20465964078903198, "p05": 0.06705628335475922, "p95": 0.3409239649772644 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.674697756767273, "group": 2, "latest_source_mass": 0.42952626943588257, "layer": 5, "max_source_mass": 0.5704737305641174, "mixer_index": 8, "normalized_entropy": 0.9733831077870527, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.5704737305641174, "median": 0.5696132779121399, "p05": 0.4695202708244324, "p95": 0.6750621199607849 }, { "label": "completed_group_1", "mean": 0.42952626943588257, "median": 0.4303867816925049, "p05": 0.3249378800392151, "p95": 0.5304797887802124 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.9691112041473389, "group": 2, "latest_source_mass": 0.2681158185005188, "layer": 5, "max_source_mass": 0.5506307482719421, "mixer_index": 9, "normalized_entropy": 0.8821230329784768, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.5506307482719421, "median": 0.543993353843689, "p05": 0.41262108087539673, "p95": 0.7161091566085815 }, { "label": "completed_group_1", "mean": 0.18125343322753906, "median": 0.1794438362121582, "p05": 0.11575750261545181, "p95": 0.25031501054763794 }, { "label": "current_group_partial", "mean": 0.2681158185005188, "median": 0.2664111256599426, "p05": 0.1564716249704361, "p95": 0.3817881643772125 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0745112895965576, "group": 2, "latest_source_mass": 0.412794291973114, "layer": 6, "max_source_mass": 0.412794291973114, "mixer_index": 10, "normalized_entropy": 0.9780623252441762, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.34262996912002563, "median": 0.3412924110889435, "p05": 0.29556700587272644, "p95": 0.39125555753707886 }, { "label": "completed_group_1", "mean": 0.24457570910453796, "median": 0.24382859468460083, "p05": 0.22120657563209534, "p95": 0.2734297513961792 }, { "label": "current_group_partial", "mean": 0.412794291973114, "median": 0.4151499271392822, "p05": 0.37735897302627563, "p95": 0.43749260902404785 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0074045658111572, "group": 2, "latest_source_mass": 0.3775783181190491, "layer": 6, "max_source_mass": 0.4399106502532959, "mixer_index": 11, "normalized_entropy": 0.9169791528842925, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.4399106502532959, "median": 0.44856905937194824, "p05": 0.22063353657722473, "p95": 0.6359403133392334 }, { "label": "completed_group_1", "mean": 0.18251106142997742, "median": 0.18562838435173035, "p05": 0.1196163073182106, "p95": 0.23045453429222107 }, { "label": "current_group_partial", "mean": 0.3775783181190491, "median": 0.3675365447998047, "p05": 0.23817461729049683, "p95": 0.5557451844215393 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.060497522354126, "group": 2, "latest_source_mass": 0.4025024175643921, "layer": 7, "max_source_mass": 0.4025024175643921, "mixer_index": 12, "normalized_entropy": 0.9653064445872968, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.3786153197288513, "median": 0.3907395303249359, "p05": 0.28039106726646423, "p95": 0.4436405599117279 }, { "label": "completed_group_1", "mean": 0.2188822478055954, "median": 0.21226292848587036, "p05": 0.1885872334241867, "p95": 0.2672030031681061 }, { "label": "current_group_partial", "mean": 0.4025024175643921, "median": 0.39815032482147217, "p05": 0.3533596992492676, "p95": 0.460100919008255 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 0.9941019415855408, "group": 2, "latest_source_mass": 0.4801652431488037, "layer": 7, "max_source_mass": 0.4801652431488037, "mixer_index": 13, "normalized_entropy": 0.9048705824970601, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.32296955585479736, "median": 0.2803739905357361, "p05": 0.1784922033548355, "p95": 0.6440245509147644 }, { "label": "completed_group_1", "mean": 0.19686520099639893, "median": 0.2053825855255127, "p05": 0.09813719242811203, "p95": 0.2504696547985077 }, { "label": "current_group_partial", "mean": 0.4801652431488037, "median": 0.5057140588760376, "p05": 0.2529003322124481, "p95": 0.598870575428009 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0503522157669067, "group": 2, "latest_source_mass": 0.4001518487930298, "layer": 8, "max_source_mass": 0.40130966901779175, "mixer_index": 14, "normalized_entropy": 0.9560717885654542, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.40130966901779175, "median": 0.396759569644928, "p05": 0.3479611873626709, "p95": 0.4733521044254303 }, { "label": "completed_group_1", "mean": 0.19853851199150085, "median": 0.19769755005836487, "p05": 0.17136667668819427, "p95": 0.22788912057876587 }, { "label": "current_group_partial", "mean": 0.4001518487930298, "median": 0.40291696786880493, "p05": 0.3506356179714203, "p95": 0.43941178917884827 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0129402875900269, "group": 2, "latest_source_mass": 0.42161279916763306, "layer": 8, "max_source_mass": 0.42161279916763306, "mixer_index": 15, "normalized_entropy": 0.9220179839951123, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.4093267023563385, "median": 0.4046899676322937, "p05": 0.2895161807537079, "p95": 0.5762131810188293 }, { "label": "completed_group_1", "mean": 0.16906049847602844, "median": 0.16888955235481262, "p05": 0.11233551055192947, "p95": 0.22477403283119202 }, { "label": "current_group_partial", "mean": 0.42161279916763306, "median": 0.4227708578109741, "p05": 0.3131049573421478, "p95": 0.519075334072113 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0774019956588745, "group": 3, "latest_source_mass": 0.3560434877872467, "layer": 9, "max_source_mass": 0.38510820269584656, "mixer_index": 16, "normalized_entropy": 0.9806935592947451, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.38510820269584656, "median": 0.3892287313938141, "p05": 0.2934730052947998, "p95": 0.4718902111053467 }, { "label": "completed_group_1", "mean": 0.25884830951690674, "median": 0.25734007358551025, "p05": 0.22217319905757904, "p95": 0.29551801085472107 }, { "label": "completed_group_2", "mean": 0.3560434877872467, "median": 0.35412120819091797, "p05": 0.29444968700408936, "p95": 0.41908568143844604 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.2740377187728882, "group": 3, "latest_source_mass": 0.3633861243724823, "layer": 9, "max_source_mass": 0.3633861243724823, "mixer_index": 17, "normalized_entropy": 0.9190239493895668, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.29550763964653015, "median": 0.30360478162765503, "p05": 0.16904757916927338, "p95": 0.45146965980529785 }, { "label": "completed_group_1", "mean": 0.09541244804859161, "median": 0.09486871212720871, "p05": 0.061360105872154236, "p95": 0.13037073612213135 }, { "label": "completed_group_2", "mean": 0.24569380283355713, "median": 0.24333015084266663, "p05": 0.1698218435049057, "p95": 0.31589236855506897 }, { "label": "current_group_partial", "mean": 0.3633861243724823, "median": 0.36472851037979126, "p05": 0.2795379161834717, "p95": 0.4399876892566681 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.360302448272705, "group": 3, "latest_source_mass": 0.28569501638412476, "layer": 10, "max_source_mass": 0.3109527826309204, "mixer_index": 18, "normalized_entropy": 0.9812507981160736, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.3109527826309204, "median": 0.32172977924346924, "p05": 0.23520725965499878, "p95": 0.37052470445632935 }, { "label": "completed_group_1", "mean": 0.20984819531440735, "median": 0.20958667993545532, "p05": 0.1891367882490158, "p95": 0.23184512555599213 }, { "label": "completed_group_2", "mean": 0.19350400567054749, "median": 0.19153150916099548, "p05": 0.16607457399368286, "p95": 0.22877082228660583 }, { "label": "current_group_partial", "mean": 0.28569501638412476, "median": 0.28060320019721985, "p05": 0.23824527859687805, "p95": 0.3439165949821472 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.2722704410552979, "group": 3, "latest_source_mass": 0.29912811517715454, "layer": 10, "max_source_mass": 0.35036465525627136, "mixer_index": 19, "normalized_entropy": 0.9177491279900463, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.35036465525627136, "median": 0.36210548877716064, "p05": 0.12807878851890564, "p95": 0.5797606706619263 }, { "label": "completed_group_1", "mean": 0.12631990015506744, "median": 0.1264108121395111, "p05": 0.06977735459804535, "p95": 0.1755298376083374 }, { "label": "completed_group_2", "mean": 0.22418735921382904, "median": 0.22563719749450684, "p05": 0.1176326721906662, "p95": 0.32001155614852905 }, { "label": "current_group_partial", "mean": 0.29912811517715454, "median": 0.28589946031570435, "p05": 0.193109393119812, "p95": 0.44119665026664734 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3558036088943481, "group": 3, "latest_source_mass": 0.314197838306427, "layer": 11, "max_source_mass": 0.314197838306427, "mixer_index": 20, "normalized_entropy": 0.978005571485618, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.298976868391037, "median": 0.2961939573287964, "p05": 0.24950163066387177, "p95": 0.35835936665534973 }, { "label": "completed_group_1", "mean": 0.1941717267036438, "median": 0.1918032318353653, "p05": 0.16809208691120148, "p95": 0.22895419597625732 }, { "label": "completed_group_2", "mean": 0.1926535815000534, "median": 0.19306811690330505, "p05": 0.16983944177627563, "p95": 0.2155935913324356 }, { "label": "current_group_partial", "mean": 0.314197838306427, "median": 0.31541913747787476, "p05": 0.2673022747039795, "p95": 0.35941022634506226 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.2696614265441895, "group": 3, "latest_source_mass": 0.2862822115421295, "layer": 11, "max_source_mass": 0.3667730689048767, "mixer_index": 21, "normalized_entropy": 0.9158671218416545, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.3667730689048767, "median": 0.4067659378051758, "p05": 0.10593366622924805, "p95": 0.5672274827957153 }, { "label": "completed_group_1", "mean": 0.15310388803482056, "median": 0.15050218999385834, "p05": 0.08651046454906464, "p95": 0.22459912300109863 }, { "label": "completed_group_2", "mean": 0.19384080171585083, "median": 0.18732509016990662, "p05": 0.1267009675502777, "p95": 0.281107634305954 }, { "label": "current_group_partial", "mean": 0.2862822115421295, "median": 0.26271265745162964, "p05": 0.17024393379688263, "p95": 0.475044310092926 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3606936931610107, "group": 3, "latest_source_mass": 0.3028184771537781, "layer": 12, "max_source_mass": 0.3028184771537781, "mixer_index": 22, "normalized_entropy": 0.9815330216461395, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.298227459192276, "median": 0.2954302132129669, "p05": 0.25287362933158875, "p95": 0.35405808687210083 }, { "label": "completed_group_1", "mean": 0.18611131608486176, "median": 0.1850002110004425, "p05": 0.15901757776737213, "p95": 0.21725836396217346 }, { "label": "completed_group_2", "mean": 0.21284276247024536, "median": 0.21402311325073242, "p05": 0.18819917738437653, "p95": 0.23418807983398438 }, { "label": "current_group_partial", "mean": 0.3028184771537781, "median": 0.3041122555732727, "p05": 0.25661808252334595, "p95": 0.34458184242248535 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.269316554069519, "group": 3, "latest_source_mass": 0.2653314769268036, "layer": 12, "max_source_mass": 0.4081852436065674, "mixer_index": 23, "normalized_entropy": 0.9156183489371815, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.4081852436065674, "median": 0.4254342317581177, "p05": 0.20765496790409088, "p95": 0.587238073348999 }, { "label": "completed_group_1", "mean": 0.14420127868652344, "median": 0.1438048928976059, "p05": 0.1064920499920845, "p95": 0.1871720254421234 }, { "label": "completed_group_2", "mean": 0.18228203058242798, "median": 0.17507661879062653, "p05": 0.12183455377817154, "p95": 0.26505547761917114 }, { "label": "current_group_partial", "mean": 0.2653314769268036, "median": 0.24924281239509583, "p05": 0.15609969198703766, "p95": 0.4061311185359955 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3636136054992676, "group": 4, "latest_source_mass": 0.3136237859725952, "layer": 13, "max_source_mass": 0.3136237859725952, "mixer_index": 24, "normalized_entropy": 0.9836392931712563, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.27427661418914795, "median": 0.27403637766838074, "p05": 0.22928935289382935, "p95": 0.319773405790329 }, { "label": "completed_group_1", "mean": 0.21341991424560547, "median": 0.20697325468063354, "p05": 0.18552586436271667, "p95": 0.26688331365585327 }, { "label": "completed_group_2", "mean": 0.19867971539497375, "median": 0.197545126080513, "p05": 0.1794913411140442, "p95": 0.222100168466568 }, { "label": "completed_group_3", "mean": 0.3136237859725952, "median": 0.31875866651535034, "p05": 0.24156072735786438, "p95": 0.3695686459541321 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.570773959159851, "group": 4, "latest_source_mass": 0.2294243425130844, "layer": 13, "max_source_mass": 0.2658740282058716, "mixer_index": 25, "normalized_entropy": 0.9759767351225285, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.2658740282058716, "median": 0.2747163474559784, "p05": 0.17927880585193634, "p95": 0.3512483835220337 }, { "label": "completed_group_1", "mean": 0.13419361412525177, "median": 0.13374894857406616, "p05": 0.11072897166013718, "p95": 0.1586270034313202 }, { "label": "completed_group_2", "mean": 0.1907704919576645, "median": 0.19074805080890656, "p05": 0.14699944853782654, "p95": 0.23316305875778198 }, { "label": "completed_group_3", "mean": 0.17973749339580536, "median": 0.17616114020347595, "p05": 0.1363743096590042, "p95": 0.23202598094940186 }, { "label": "current_group_partial", "mean": 0.2294243425130844, "median": 0.22995705902576447, "p05": 0.17975632846355438, "p95": 0.2789157032966614 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.5560038089752197, "group": 4, "latest_source_mass": 0.32468360662460327, "layer": 14, "max_source_mass": 0.32468360662460327, "mixer_index": 26, "normalized_entropy": 0.9667995248241249, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.20477107167243958, "median": 0.206403911113739, "p05": 0.16619175672531128, "p95": 0.2431035190820694 }, { "label": "completed_group_1", "mean": 0.15721669793128967, "median": 0.15658551454544067, "p05": 0.13878680765628815, "p95": 0.17778633534908295 }, { "label": "completed_group_2", "mean": 0.13811248540878296, "median": 0.13743364810943604, "p05": 0.12352626770734787, "p95": 0.15553249418735504 }, { "label": "completed_group_3", "mean": 0.1752161681652069, "median": 0.17309701442718506, "p05": 0.15223677456378937, "p95": 0.20474471151828766 }, { "label": "current_group_partial", "mean": 0.32468360662460327, "median": 0.32497870922088623, "p05": 0.2946181893348694, "p95": 0.35203880071640015 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.5659973621368408, "group": 4, "latest_source_mass": 0.26703086495399475, "layer": 14, "max_source_mass": 0.26703086495399475, "mixer_index": 27, "normalized_entropy": 0.9730088685238187, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.20531430840492249, "median": 0.20229148864746094, "p05": 0.10698921978473663, "p95": 0.3393132984638214 }, { "label": "completed_group_1", "mean": 0.15986156463623047, "median": 0.15952220559120178, "p05": 0.11969029903411865, "p95": 0.20180393755435944 }, { "label": "completed_group_2", "mean": 0.21557100117206573, "median": 0.21654027700424194, "p05": 0.1673085242509842, "p95": 0.2610440254211426 }, { "label": "completed_group_3", "mean": 0.15222224593162537, "median": 0.1510905921459198, "p05": 0.11800260841846466, "p95": 0.18893754482269287 }, { "label": "current_group_partial", "mean": 0.26703086495399475, "median": 0.258539617061615, "p05": 0.19330690801143646, "p95": 0.36521556973457336 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.5552496910095215, "group": 4, "latest_source_mass": 0.31774693727493286, "layer": 15, "max_source_mass": 0.31774693727493286, "mixer_index": 28, "normalized_entropy": 0.9663309649872576, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.2162160575389862, "median": 0.2145705670118332, "p05": 0.17727568745613098, "p95": 0.26878347992897034 }, { "label": "completed_group_1", "mean": 0.17982497811317444, "median": 0.18024154007434845, "p05": 0.15453973412513733, "p95": 0.2030555009841919 }, { "label": "completed_group_2", "mean": 0.1307673156261444, "median": 0.13107559084892273, "p05": 0.11463287472724915, "p95": 0.14630235731601715 }, { "label": "completed_group_3", "mean": 0.1554446816444397, "median": 0.15516987442970276, "p05": 0.13627474009990692, "p95": 0.1754152774810791 }, { "label": "current_group_partial", "mean": 0.31774693727493286, "median": 0.319039523601532, "p05": 0.27606821060180664, "p95": 0.35650834441185 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.5068919658660889, "group": 4, "latest_source_mass": 0.3298327624797821, "layer": 15, "max_source_mass": 0.3298327624797821, "mixer_index": 29, "normalized_entropy": 0.9362846209998111, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.20646697282791138, "median": 0.18775638937950134, "p05": 0.0806753933429718, "p95": 0.4274584949016571 }, { "label": "completed_group_1", "mean": 0.1197461187839508, "median": 0.11989086866378784, "p05": 0.0809398666024208, "p95": 0.15743419528007507 }, { "label": "completed_group_2", "mean": 0.19215428829193115, "median": 0.19620640575885773, "p05": 0.1148109883069992, "p95": 0.2491212636232376 }, { "label": "completed_group_3", "mean": 0.15179988741874695, "median": 0.15212899446487427, "p05": 0.10697861760854721, "p95": 0.19542087614536285 }, { "label": "current_group_partial", "mean": 0.3298327624797821, "median": 0.32837340235710144, "p05": 0.22746729850769043, "p95": 0.4339301884174347 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.5435223579406738, "group": 4, "latest_source_mass": 0.3224271535873413, "layer": 16, "max_source_mass": 0.3224271535873413, "mixer_index": 30, "normalized_entropy": 0.9590443632623663, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.22410975396633148, "median": 0.22760415077209473, "p05": 0.14593973755836487, "p95": 0.30093055963516235 }, { "label": "completed_group_1", "mean": 0.16986176371574402, "median": 0.16963240504264832, "p05": 0.13569067418575287, "p95": 0.20651353895664215 }, { "label": "completed_group_2", "mean": 0.13080403208732605, "median": 0.13075244426727295, "p05": 0.10945598781108856, "p95": 0.15187400579452515 }, { "label": "completed_group_3", "mean": 0.15279728174209595, "median": 0.15036612749099731, "p05": 0.12810495495796204, "p95": 0.18468116223812103 }, { "label": "current_group_partial", "mean": 0.3224271535873413, "median": 0.3164973258972168, "p05": 0.2600978910923004, "p95": 0.40530598163604736 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.4949920177459717, "group": 4, "latest_source_mass": 0.31847020983695984, "layer": 16, "max_source_mass": 0.31847020983695984, "mixer_index": 31, "normalized_entropy": 0.9288907675133354, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.27174079418182373, "median": 0.2585451304912567, "p05": 0.13639721274375916, "p95": 0.4698348045349121 }, { "label": "completed_group_1", "mean": 0.12178683280944824, "median": 0.11714950203895569, "p05": 0.08594125509262085, "p95": 0.1723237782716751 }, { "label": "completed_group_2", "mean": 0.1529826819896698, "median": 0.15223106741905212, "p05": 0.10371473431587219, "p95": 0.203575000166893 }, { "label": "completed_group_3", "mean": 0.1350194811820984, "median": 0.13363957405090332, "p05": 0.08964549005031586, "p95": 0.18468207120895386 }, { "label": "current_group_partial", "mean": 0.31847020983695984, "median": 0.3185216188430786, "p05": 0.21444711089134216, "p95": 0.4301546812057495 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.544264793395996, "group": 5, "latest_source_mass": 0.2861037850379944, "layer": 17, "max_source_mass": 0.2861037850379944, "mixer_index": 32, "normalized_entropy": 0.9595056643474137, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.24016115069389343, "median": 0.23868846893310547, "p05": 0.18435582518577576, "p95": 0.2983453571796417 }, { "label": "completed_group_1", "mean": 0.20730282366275787, "median": 0.1998077780008316, "p05": 0.16255050897598267, "p95": 0.28107208013534546 }, { "label": "completed_group_2", "mean": 0.09692573547363281, "median": 0.0967404842376709, "p05": 0.08098706603050232, "p95": 0.11221462488174438 }, { "label": "completed_group_3", "mean": 0.1695065200328827, "median": 0.1696585714817047, "p05": 0.136396586894989, "p95": 0.20130634307861328 }, { "label": "completed_group_4", "mean": 0.2861037850379944, "median": 0.28883200883865356, "p05": 0.22401469945907593, "p95": 0.3409370183944702 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.686682939529419, "group": 5, "latest_source_mass": 0.12281867861747742, "layer": 17, "max_source_mass": 0.2885604202747345, "mixer_index": 33, "normalized_entropy": 0.9413556721740635, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.16755425930023193, "median": 0.1327776312828064, "p05": 0.0790192037820816, "p95": 0.3667412996292114 }, { "label": "completed_group_1", "mean": 0.10962115228176117, "median": 0.10525335371494293, "p05": 0.07840172946453094, "p95": 0.15405339002609253 }, { "label": "completed_group_2", "mean": 0.1783454418182373, "median": 0.17896884679794312, "p05": 0.10802385956048965, "p95": 0.24048911035060883 }, { "label": "completed_group_3", "mean": 0.13310004770755768, "median": 0.13033035397529602, "p05": 0.0904831737279892, "p95": 0.17934636771678925 }, { "label": "completed_group_4", "mean": 0.2885604202747345, "median": 0.2831825315952301, "p05": 0.16967125236988068, "p95": 0.4189499616622925 }, { "label": "current_group_partial", "mean": 0.12281867861747742, "median": 0.12062528729438782, "p05": 0.07461707293987274, "p95": 0.1782219111919403 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.7070834636688232, "group": 5, "latest_source_mass": 0.2988377809524536, "layer": 18, "max_source_mass": 0.2988377809524536, "mixer_index": 34, "normalized_entropy": 0.9527414214834803, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.181177020072937, "median": 0.18419909477233887, "p05": 0.13624237477779388, "p95": 0.22192169725894928 }, { "label": "completed_group_1", "mean": 0.14557164907455444, "median": 0.144476979970932, "p05": 0.1293315887451172, "p95": 0.16486899554729462 }, { "label": "completed_group_2", "mean": 0.08318156749010086, "median": 0.0820261612534523, "p05": 0.07088497281074524, "p95": 0.0990293100476265 }, { "label": "completed_group_3", "mean": 0.11424209177494049, "median": 0.11267061531543732, "p05": 0.0979941189289093, "p95": 0.1353992223739624 }, { "label": "completed_group_4", "mean": 0.17698991298675537, "median": 0.17597860097885132, "p05": 0.14837387204170227, "p95": 0.21050144731998444 }, { "label": "current_group_partial", "mean": 0.2988377809524536, "median": 0.2989804744720459, "p05": 0.2721370458602905, "p95": 0.32572072744369507 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.7419235706329346, "group": 5, "latest_source_mass": 0.26016175746917725, "layer": 18, "max_source_mass": 0.26016175746917725, "mixer_index": 35, "normalized_entropy": 0.972186055410333, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1532188355922699, "median": 0.1451343297958374, "p05": 0.11514341086149216, "p95": 0.22103679180145264 }, { "label": "completed_group_1", "mean": 0.11420352011919022, "median": 0.11261284351348877, "p05": 0.0928458571434021, "p95": 0.14123988151550293 }, { "label": "completed_group_2", "mean": 0.16597424447536469, "median": 0.1655328869819641, "p05": 0.13871800899505615, "p95": 0.19611573219299316 }, { "label": "completed_group_3", "mean": 0.1250365972518921, "median": 0.12369821965694427, "p05": 0.10375828295946121, "p95": 0.14912742376327515 }, { "label": "completed_group_4", "mean": 0.18140508234500885, "median": 0.1796584278345108, "p05": 0.14511024951934814, "p95": 0.22286373376846313 }, { "label": "current_group_partial", "mean": 0.26016175746917725, "median": 0.26266711950302124, "p05": 0.1889389455318451, "p95": 0.3234381675720215 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.7127439975738525, "group": 5, "latest_source_mass": 0.2907913327217102, "layer": 19, "max_source_mass": 0.2907913327217102, "mixer_index": 36, "normalized_entropy": 0.9559006256078307, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.18731768429279327, "median": 0.19042012095451355, "p05": 0.13879702985286713, "p95": 0.23792000114917755 }, { "label": "completed_group_1", "mean": 0.16715484857559204, "median": 0.1667211651802063, "p05": 0.14574480056762695, "p95": 0.19026139378547668 }, { "label": "completed_group_2", "mean": 0.08496320247650146, "median": 0.08452782034873962, "p05": 0.07410907745361328, "p95": 0.097386933863163 }, { "label": "completed_group_3", "mean": 0.11362075805664062, "median": 0.11248823255300522, "p05": 0.0977015346288681, "p95": 0.13219426572322845 }, { "label": "completed_group_4", "mean": 0.15615218877792358, "median": 0.15540224313735962, "p05": 0.1289147436618805, "p95": 0.18598109483718872 }, { "label": "current_group_partial", "mean": 0.2907913327217102, "median": 0.2900415062904358, "p05": 0.25822532176971436, "p95": 0.32475030422210693 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.6076834201812744, "group": 5, "latest_source_mass": 0.3738824725151062, "layer": 19, "max_source_mass": 0.3738824725151062, "mixer_index": 37, "normalized_entropy": 0.8972652009334232, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.15032559633255005, "median": 0.12544110417366028, "p05": 0.06976518034934998, "p95": 0.35840943455696106 }, { "label": "completed_group_1", "mean": 0.08624927699565887, "median": 0.08602266013622284, "p05": 0.061998721212148666, "p95": 0.11138717085123062 }, { "label": "completed_group_2", "mean": 0.15736976265907288, "median": 0.15802210569381714, "p05": 0.10121746361255646, "p95": 0.20687000453472137 }, { "label": "completed_group_3", "mean": 0.08584291487932205, "median": 0.0827617198228836, "p05": 0.060561805963516235, "p95": 0.11351694166660309 }, { "label": "completed_group_4", "mean": 0.14632999897003174, "median": 0.1426895558834076, "p05": 0.10106660425662994, "p95": 0.20179018378257751 }, { "label": "current_group_partial", "mean": 0.3738824725151062, "median": 0.3799295425415039, "p05": 0.23631010949611664, "p95": 0.4884527325630188 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.709290862083435, "group": 5, "latest_source_mass": 0.2901708781719208, "layer": 20, "max_source_mass": 0.2901708781719208, "mixer_index": 38, "normalized_entropy": 0.9539733939957076, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.18991519510746002, "median": 0.1924753338098526, "p05": 0.14201581478118896, "p95": 0.23709386587142944 }, { "label": "completed_group_1", "mean": 0.17091524600982666, "median": 0.17006167769432068, "p05": 0.15297070145606995, "p95": 0.19153237342834473 }, { "label": "completed_group_2", "mean": 0.07822999358177185, "median": 0.07778562605381012, "p05": 0.06707640737295151, "p95": 0.09089364111423492 }, { "label": "completed_group_3", "mean": 0.116765096783638, "median": 0.11516137421131134, "p05": 0.10162825882434845, "p95": 0.13653048872947693 }, { "label": "completed_group_4", "mean": 0.1540035903453827, "median": 0.15287795662879944, "p05": 0.12593276798725128, "p95": 0.18590238690376282 }, { "label": "current_group_partial", "mean": 0.2901708781719208, "median": 0.29104921221733093, "p05": 0.25643545389175415, "p95": 0.3205650746822357 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.6690887212753296, "group": 5, "latest_source_mass": 0.3342471718788147, "layer": 20, "max_source_mass": 0.3342471718788147, "mixer_index": 39, "normalized_entropy": 0.9315361520005944, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.19029226899147034, "median": 0.19644704461097717, "p05": 0.09456793963909149, "p95": 0.30468082427978516 }, { "label": "completed_group_1", "mean": 0.1100241169333458, "median": 0.10808776319026947, "p05": 0.09035106748342514, "p95": 0.1359245330095291 }, { "label": "completed_group_2", "mean": 0.11471512168645859, "median": 0.11494918912649155, "p05": 0.08799737691879272, "p95": 0.14024731516838074 }, { "label": "completed_group_3", "mean": 0.09999909996986389, "median": 0.09682698547840118, "p05": 0.07921088486909866, "p95": 0.1264391839504242 }, { "label": "completed_group_4", "mean": 0.1507222205400467, "median": 0.1476232260465622, "p05": 0.11397356539964676, "p95": 0.19991227984428406 }, { "label": "current_group_partial", "mean": 0.3342471718788147, "median": 0.33802270889282227, "p05": 0.2548266053199768, "p95": 0.40130624175071716 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.7458693981170654, "group": 6, "latest_source_mass": 0.2361394464969635, "layer": 21, "max_source_mass": 0.2361394464969635, "mixer_index": 40, "normalized_entropy": 0.9743882636597644, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.20528709888458252, "median": 0.2140858769416809, "p05": 0.13168928027153015, "p95": 0.27098023891448975 }, { "label": "completed_group_1", "mean": 0.1722964197397232, "median": 0.17238952219486237, "p05": 0.15053534507751465, "p95": 0.19365043938159943 }, { "label": "completed_group_2", "mean": 0.10435444861650467, "median": 0.10361101478338242, "p05": 0.08888213336467743, "p95": 0.12212536484003067 }, { "label": "completed_group_3", "mean": 0.12075909972190857, "median": 0.1180175170302391, "p05": 0.10358668118715286, "p95": 0.14458929002285004 }, { "label": "completed_group_4", "mean": 0.16116347908973694, "median": 0.15841388702392578, "p05": 0.1337081342935562, "p95": 0.19791515171527863 }, { "label": "completed_group_5", "mean": 0.2361394464969635, "median": 0.23709020018577576, "p05": 0.20446214079856873, "p95": 0.26509594917297363 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.4612125158309937, "group": 6, "latest_source_mass": 0.5543455481529236, "layer": 21, "max_source_mass": 0.5543455481529236, "mixer_index": 41, "normalized_entropy": 0.7509146897354808, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.08934466540813446, "median": 0.07911523431539536, "p05": 0.04488275945186615, "p95": 0.1737232506275177 }, { "label": "completed_group_1", "mean": 0.054379627108573914, "median": 0.05344422906637192, "p05": 0.04221750423312187, "p95": 0.06924191862344742 }, { "label": "completed_group_2", "mean": 0.07567165046930313, "median": 0.07438549399375916, "p05": 0.05811355262994766, "p95": 0.09632256627082825 }, { "label": "completed_group_3", "mean": 0.05758609622716904, "median": 0.05612001568078995, "p05": 0.044792283326387405, "p95": 0.07347827404737473 }, { "label": "completed_group_4", "mean": 0.07884950935840607, "median": 0.07757881283760071, "p05": 0.05758974328637123, "p95": 0.10452556610107422 }, { "label": "completed_group_5", "mean": 0.08982288837432861, "median": 0.08912864327430725, "p05": 0.0663854256272316, "p95": 0.11604885756969452 }, { "label": "current_group_partial", "mean": 0.5543455481529236, "median": 0.5596089363098145, "p05": 0.45455610752105713, "p95": 0.6367484331130981 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.7241814136505127, "group": 6, "latest_source_mass": 0.39484071731567383, "layer": 22, "max_source_mass": 0.39484071731567383, "mixer_index": 42, "normalized_entropy": 0.886053970419732, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.13360010087490082, "median": 0.14072942733764648, "p05": 0.07699814438819885, "p95": 0.17506597936153412 }, { "label": "completed_group_1", "mean": 0.10609637200832367, "median": 0.10586297512054443, "p05": 0.08648421615362167, "p95": 0.12602579593658447 }, { "label": "completed_group_2", "mean": 0.06363911926746368, "median": 0.06324908882379532, "p05": 0.054980404675006866, "p95": 0.07316673547029495 }, { "label": "completed_group_3", "mean": 0.07426095008850098, "median": 0.07362888753414154, "p05": 0.06318790465593338, "p95": 0.08665015548467636 }, { "label": "completed_group_4", "mean": 0.09383076429367065, "median": 0.09294568747282028, "p05": 0.07638302445411682, "p95": 0.11445100605487823 }, { "label": "completed_group_5", "mean": 0.13373197615146637, "median": 0.13327741622924805, "p05": 0.11399447172880173, "p95": 0.15641780197620392 }, { "label": "current_group_partial", "mean": 0.39484071731567383, "median": 0.3879151940345764, "p05": 0.3302172124385834, "p95": 0.4852263331413269 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.3979300260543823, "group": 6, "latest_source_mass": 0.5830056071281433, "layer": 22, "max_source_mass": 0.5830056071281433, "mixer_index": 43, "normalized_entropy": 0.7183939231382495, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.08959861099720001, "median": 0.07544538378715515, "p05": 0.04165133833885193, "p95": 0.20099139213562012 }, { "label": "completed_group_1", "mean": 0.05414731800556183, "median": 0.05336771905422211, "p05": 0.04069708660244942, "p95": 0.07046990841627121 }, { "label": "completed_group_2", "mean": 0.07668881118297577, "median": 0.07607124000787735, "p05": 0.0555989071726799, "p95": 0.09980960935354233 }, { "label": "completed_group_3", "mean": 0.05603286623954773, "median": 0.054573021829128265, "p05": 0.04162490367889404, "p95": 0.0730118378996849 }, { "label": "completed_group_4", "mean": 0.06979228556156158, "median": 0.06884932518005371, "p05": 0.04576495662331581, "p95": 0.09806846082210541 }, { "label": "completed_group_5", "mean": 0.07073447108268738, "median": 0.07023533433675766, "p05": 0.04956750199198723, "p95": 0.09379896521568298 }, { "label": "current_group_partial", "mean": 0.5830056071281433, "median": 0.5862616896629333, "p05": 0.4658474922180176, "p95": 0.6849814057350159 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.7459560632705688, "group": 6, "latest_source_mass": 0.37691548466682434, "layer": 23, "max_source_mass": 0.37691548466682434, "mixer_index": 44, "normalized_entropy": 0.897243926765161, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.14318783581256866, "median": 0.1493408977985382, "p05": 0.08292607218027115, "p95": 0.19464610517024994 }, { "label": "completed_group_1", "mean": 0.1094653457403183, "median": 0.10858799517154694, "p05": 0.09432064741849899, "p95": 0.1279931515455246 }, { "label": "completed_group_2", "mean": 0.06349228322505951, "median": 0.06325754523277283, "p05": 0.05439673364162445, "p95": 0.0737837627530098 }, { "label": "completed_group_3", "mean": 0.07859592139720917, "median": 0.0775163471698761, "p05": 0.06603655964136124, "p95": 0.09357087314128876 }, { "label": "completed_group_4", "mean": 0.09567482024431229, "median": 0.09468592703342438, "p05": 0.07671141624450684, "p95": 0.11774884164333344 }, { "label": "completed_group_5", "mean": 0.13266831636428833, "median": 0.13199329376220703, "p05": 0.11244359612464905, "p95": 0.15504513680934906 }, { "label": "current_group_partial", "mean": 0.37691548466682434, "median": 0.37180638313293457, "p05": 0.31207239627838135, "p95": 0.4595233201980591 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.4055393934249878, "group": 6, "latest_source_mass": 0.5772827863693237, "layer": 23, "max_source_mass": 0.5772827863693237, "mixer_index": 45, "normalized_entropy": 0.7223043644164862, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.10154955089092255, "median": 0.08766824007034302, "p05": 0.04901552200317383, "p95": 0.20287738740444183 }, { "label": "completed_group_1", "mean": 0.05619276314973831, "median": 0.055477917194366455, "p05": 0.041608311235904694, "p95": 0.07371112704277039 }, { "label": "completed_group_2", "mean": 0.07791627198457718, "median": 0.07736755907535553, "p05": 0.055920060724020004, "p95": 0.10158214718103409 }, { "label": "completed_group_3", "mean": 0.053915880620479584, "median": 0.05215863138437271, "p05": 0.03798118978738785, "p95": 0.07156119495630264 }, { "label": "completed_group_4", "mean": 0.06398355215787888, "median": 0.06229078769683838, "p05": 0.040616974234580994, "p95": 0.09270931780338287 }, { "label": "completed_group_5", "mean": 0.06915920227766037, "median": 0.0684593990445137, "p05": 0.046986356377601624, "p95": 0.09408729523420334 }, { "label": "current_group_partial", "mean": 0.5772827863693237, "median": 0.5824689269065857, "p05": 0.4518115520477295, "p95": 0.6926202178001404 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.7853620052337646, "group": 6, "latest_source_mass": 0.3407441973686218, "layer": 24, "max_source_mass": 0.3407441973686218, "mixer_index": 46, "normalized_entropy": 0.9174945750195659, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.15284113585948944, "median": 0.1593906283378601, "p05": 0.08678612112998962, "p95": 0.211342453956604 }, { "label": "completed_group_1", "mean": 0.12206703424453735, "median": 0.12153112143278122, "p05": 0.1060115247964859, "p95": 0.1399766057729721 }, { "label": "completed_group_2", "mean": 0.06508444249629974, "median": 0.06482555717229843, "p05": 0.055350616574287415, "p95": 0.07557765394449234 }, { "label": "completed_group_3", "mean": 0.08347313106060028, "median": 0.08190013468265533, "p05": 0.06966532766819, "p95": 0.10050469636917114 }, { "label": "completed_group_4", "mean": 0.09811970591545105, "median": 0.09664031863212585, "p05": 0.07688035070896149, "p95": 0.12369741499423981 }, { "label": "completed_group_5", "mean": 0.13767032325267792, "median": 0.1373325139284134, "p05": 0.11642646044492722, "p95": 0.1606694608926773 }, { "label": "current_group_partial", "mean": 0.3407441973686218, "median": 0.33805108070373535, "p05": 0.271301805973053, "p95": 0.41781002283096313 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.509842872619629, "group": 6, "latest_source_mass": 0.5227814316749573, "layer": 24, "max_source_mass": 0.5227814316749573, "mixer_index": 47, "normalized_entropy": 0.7759057494780099, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.11788781732320786, "median": 0.10097194463014603, "p05": 0.06265684217214584, "p95": 0.2408325970172882 }, { "label": "completed_group_1", "mean": 0.06822571158409119, "median": 0.0673636943101883, "p05": 0.05209728330373764, "p95": 0.08648915588855743 }, { "label": "completed_group_2", "mean": 0.08556114137172699, "median": 0.08463940769433975, "p05": 0.06234566867351532, "p95": 0.1125786080956459 }, { "label": "completed_group_3", "mean": 0.06067090481519699, "median": 0.058974385261535645, "p05": 0.043267179280519485, "p95": 0.07997901737689972 }, { "label": "completed_group_4", "mean": 0.07228855788707733, "median": 0.06972275674343109, "p05": 0.04689513146877289, "p95": 0.106512151658535 }, { "label": "completed_group_5", "mean": 0.07258446514606476, "median": 0.07082539796829224, "p05": 0.050949547439813614, "p95": 0.09970614314079285 }, { "label": "current_group_partial", "mean": 0.5227814316749573, "median": 0.5290273427963257, "p05": 0.38773614168167114, "p95": 0.6392589807510376 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.839367389678955, "group": 7, "latest_source_mass": 0.29583215713500977, "layer": 25, "max_source_mass": 0.29583215713500977, "mixer_index": 48, "normalized_entropy": 0.9452478525649903, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.15906387567520142, "median": 0.15846745669841766, "p05": 0.09519976377487183, "p95": 0.24517810344696045 }, { "label": "completed_group_1", "mean": 0.13139380514621735, "median": 0.1308426856994629, "p05": 0.11478274315595627, "p95": 0.15000829100608826 }, { "label": "completed_group_2", "mean": 0.07717745751142502, "median": 0.07715500891208649, "p05": 0.06616044044494629, "p95": 0.08791667222976685 }, { "label": "completed_group_3", "mean": 0.09607303142547607, "median": 0.09436552226543427, "p05": 0.08152313530445099, "p95": 0.11384639143943787 }, { "label": "completed_group_4", "mean": 0.1107850894331932, "median": 0.1089654266834259, "p05": 0.08814431726932526, "p95": 0.139452263712883 }, { "label": "completed_group_5", "mean": 0.12967455387115479, "median": 0.1299237608909607, "p05": 0.10832332074642181, "p95": 0.15138982236385345 }, { "label": "completed_group_6", "mean": 0.29583215713500977, "median": 0.29647016525268555, "p05": 0.2276897132396698, "p95": 0.3604946434497833 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.5841834545135498, "group": 7, "latest_source_mass": 0.40767741203308105, "layer": 25, "max_source_mass": 0.40767741203308105, "mixer_index": 49, "normalized_entropy": 0.7618312045616817, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.07896879315376282, "median": 0.06809917092323303, "p05": 0.025932887569069862, "p95": 0.19368338584899902 }, { "label": "completed_group_1", "mean": 0.048752427101135254, "median": 0.04767637699842453, "p05": 0.034443892538547516, "p95": 0.0667153000831604 }, { "label": "completed_group_2", "mean": 0.055707164108753204, "median": 0.0549345538020134, "p05": 0.038696639239788055, "p95": 0.0750277042388916 }, { "label": "completed_group_3", "mean": 0.04139826446771622, "median": 0.03878612071275711, "p05": 0.026673054322600365, "p95": 0.05746251344680786 }, { "label": "completed_group_4", "mean": 0.04449018836021423, "median": 0.04196460545063019, "p05": 0.026162121444940567, "p95": 0.07164917886257172 }, { "label": "completed_group_5", "mean": 0.04285799711942673, "median": 0.04109475016593933, "p05": 0.029393533244729042, "p95": 0.06277710199356079 }, { "label": "completed_group_6", "mean": 0.2801477909088135, "median": 0.2763082981109619, "p05": 0.1455708146095276, "p95": 0.4254714846611023 }, { "label": "current_group_partial", "mean": 0.40767741203308105, "median": 0.4120897948741913, "p05": 0.21399420499801636, "p95": 0.5853031277656555 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.8125314712524414, "group": 7, "latest_source_mass": 0.3700707256793976, "layer": 26, "max_source_mass": 0.3700707256793976, "mixer_index": 50, "normalized_entropy": 0.8716433883436914, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.10889513790607452, "median": 0.11333495378494263, "p05": 0.04850132763385773, "p95": 0.1620434820652008 }, { "label": "completed_group_1", "mean": 0.08656036853790283, "median": 0.08640051633119583, "p05": 0.0650845319032669, "p95": 0.1084962785243988 }, { "label": "completed_group_2", "mean": 0.048123255372047424, "median": 0.04800213500857353, "p05": 0.03576180338859558, "p95": 0.06107594817876816 }, { "label": "completed_group_3", "mean": 0.06561170518398285, "median": 0.06379580497741699, "p05": 0.050968848168849945, "p95": 0.08332888036966324 }, { "label": "completed_group_4", "mean": 0.06917259097099304, "median": 0.06724178791046143, "p05": 0.050207968801259995, "p95": 0.09373342245817184 }, { "label": "completed_group_5", "mean": 0.08286689221858978, "median": 0.08183546364307404, "p05": 0.06535022705793381, "p95": 0.10340933501720428 }, { "label": "completed_group_6", "mean": 0.16869932413101196, "median": 0.16509854793548584, "p05": 0.11059704422950745, "p95": 0.23654375970363617 }, { "label": "current_group_partial", "mean": 0.3700707256793976, "median": 0.3694133758544922, "p05": 0.23441238701343536, "p95": 0.5025027990341187 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.50089693069458, "group": 7, "latest_source_mass": 0.49096518754959106, "layer": 26, "max_source_mass": 0.49096518754959106, "mixer_index": 51, "normalized_entropy": 0.7217788529328457, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.07974998652935028, "median": 0.07402504235506058, "p05": 0.04579866677522659, "p95": 0.1359037309885025 }, { "label": "completed_group_1", "mean": 0.04166398197412491, "median": 0.04038450866937637, "p05": 0.027992289513349533, "p95": 0.05969971418380737 }, { "label": "completed_group_2", "mean": 0.04607272148132324, "median": 0.044438350945711136, "p05": 0.030038591474294662, "p95": 0.06808309257030487 }, { "label": "completed_group_3", "mean": 0.03586764633655548, "median": 0.03317485377192497, "p05": 0.02203454077243805, "p95": 0.05572832003235817 }, { "label": "completed_group_4", "mean": 0.04223340004682541, "median": 0.03930673003196716, "p05": 0.02465122565627098, "p95": 0.06968094408512115 }, { "label": "completed_group_5", "mean": 0.04095568507909775, "median": 0.03919431194663048, "p05": 0.026239857077598572, "p95": 0.06058088690042496 }, { "label": "completed_group_6", "mean": 0.22249138355255127, "median": 0.2125401496887207, "p05": 0.10647299885749817, "p95": 0.3711881935596466 }, { "label": "current_group_partial", "mean": 0.49096518754959106, "median": 0.4988659620285034, "p05": 0.2878011167049408, "p95": 0.6676807403564453 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.8059929609298706, "group": 7, "latest_source_mass": 0.37139976024627686, "layer": 27, "max_source_mass": 0.37139976024627686, "mixer_index": 52, "normalized_entropy": 0.8684990295379666, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.10711923241615295, "median": 0.10972198843955994, "p05": 0.04896187782287598, "p95": 0.16201043128967285 }, { "label": "completed_group_1", "mean": 0.07651707530021667, "median": 0.07666032761335373, "p05": 0.05141128972172737, "p95": 0.10152198374271393 }, { "label": "completed_group_2", "mean": 0.058284737169742584, "median": 0.05812416970729828, "p05": 0.039273619651794434, "p95": 0.07745087146759033 }, { "label": "completed_group_3", "mean": 0.06609587371349335, "median": 0.06383530795574188, "p05": 0.04481930285692215, "p95": 0.09336265176534653 }, { "label": "completed_group_4", "mean": 0.07139429450035095, "median": 0.0694514587521553, "p05": 0.0485839769244194, "p95": 0.10034126043319702 }, { "label": "completed_group_5", "mean": 0.07493151724338531, "median": 0.07319007813930511, "p05": 0.051876723766326904, "p95": 0.10402432829141617 }, { "label": "completed_group_6", "mean": 0.17425751686096191, "median": 0.16937679052352905, "p05": 0.10449051856994629, "p95": 0.2624811828136444 }, { "label": "current_group_partial", "mean": 0.37139976024627686, "median": 0.3732905089855194, "p05": 0.23830299079418182, "p95": 0.5014542937278748 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.5622732639312744, "group": 7, "latest_source_mass": 0.4768889844417572, "layer": 27, "max_source_mass": 0.4768889844417572, "mixer_index": 53, "normalized_entropy": 0.7512946301290215, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.08775827288627625, "median": 0.08504067361354828, "p05": 0.0428270623087883, "p95": 0.144285649061203 }, { "label": "completed_group_1", "mean": 0.047765426337718964, "median": 0.04646281152963638, "p05": 0.031814731657505035, "p95": 0.06807165592908859 }, { "label": "completed_group_2", "mean": 0.053928617388010025, "median": 0.052262187004089355, "p05": 0.035437777638435364, "p95": 0.0784272700548172 }, { "label": "completed_group_3", "mean": 0.04293380305171013, "median": 0.03996016085147858, "p05": 0.02556057646870613, "p95": 0.06623870134353638 }, { "label": "completed_group_4", "mean": 0.04174989461898804, "median": 0.03903091698884964, "p05": 0.025013400241732597, "p95": 0.06869364529848099 }, { "label": "completed_group_5", "mean": 0.04734005779027939, "median": 0.04609068110585213, "p05": 0.029884062707424164, "p95": 0.06911227852106094 }, { "label": "completed_group_6", "mean": 0.20163494348526, "median": 0.19222964346408844, "p05": 0.09963390231132507, "p95": 0.3307305574417114 }, { "label": "current_group_partial", "mean": 0.4768889844417572, "median": 0.48383551836013794, "p05": 0.276212215423584, "p95": 0.6487575173377991 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.8757455348968506, "group": 7, "latest_source_mass": 0.3109493851661682, "layer": 28, "max_source_mass": 0.3109493851661682, "mixer_index": 54, "normalized_entropy": 0.9020429270551009, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.11563669890165329, "median": 0.12087491154670715, "p05": 0.05733206868171692, "p95": 0.1688953936100006 }, { "label": "completed_group_1", "mean": 0.08108697831630707, "median": 0.08185897022485733, "p05": 0.05175650864839554, "p95": 0.10753051191568375 }, { "label": "completed_group_2", "mean": 0.07280858606100082, "median": 0.07357072830200195, "p05": 0.04870978742837906, "p95": 0.0948689877986908 }, { "label": "completed_group_3", "mean": 0.07553227245807648, "median": 0.07362619787454605, "p05": 0.056317005306482315, "p95": 0.09756095707416534 }, { "label": "completed_group_4", "mean": 0.07606014609336853, "median": 0.07456053793430328, "p05": 0.05528895929455757, "p95": 0.1022825762629509 }, { "label": "completed_group_5", "mean": 0.06935927271842957, "median": 0.06843917071819305, "p05": 0.04978285729885101, "p95": 0.09167985618114471 }, { "label": "completed_group_6", "mean": 0.19856664538383484, "median": 0.19508783519268036, "p05": 0.1281973123550415, "p95": 0.27796900272369385 }, { "label": "current_group_partial", "mean": 0.3109493851661682, "median": 0.3105033040046692, "p05": 0.18172834813594818, "p95": 0.43916669487953186 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.6924852132797241, "group": 7, "latest_source_mass": 0.3315581679344177, "layer": 28, "max_source_mass": 0.3315581679344177, "mixer_index": 55, "normalized_entropy": 0.8139133413255193, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.08638577908277512, "median": 0.058608733117580414, "p05": 0.026544082909822464, "p95": 0.2740764319896698 }, { "label": "completed_group_1", "mean": 0.059912845492362976, "median": 0.05901291221380234, "p05": 0.03629481792449951, "p95": 0.08707985281944275 }, { "label": "completed_group_2", "mean": 0.06609142571687698, "median": 0.0647939145565033, "p05": 0.038691695779561996, "p95": 0.09703366458415985 }, { "label": "completed_group_3", "mean": 0.056745611131191254, "median": 0.05418796092271805, "p05": 0.032174739986658096, "p95": 0.08370660990476608 }, { "label": "completed_group_4", "mean": 0.06277773529291153, "median": 0.0600711889564991, "p05": 0.03501531109213829, "p95": 0.10094858705997467 }, { "label": "completed_group_5", "mean": 0.05352145433425903, "median": 0.05179818347096443, "p05": 0.032271772623062134, "p95": 0.08084158599376678 }, { "label": "completed_group_6", "mean": 0.2830069661140442, "median": 0.2759752869606018, "p05": 0.12981083989143372, "p95": 0.46204638481140137 }, { "label": "current_group_partial", "mean": 0.3315581679344177, "median": 0.3204721510410309, "p05": 0.13446375727653503, "p95": 0.5667697787284851 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.8980522155761719, "group": 8, "latest_source_mass": 0.20123262703418732, "layer": 29, "max_source_mass": 0.26602888107299805, "mixer_index": 56, "normalized_entropy": 0.9127701729200177, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.11603894829750061, "median": 0.09991352260112762, "p05": 0.030035395175218582, "p95": 0.2941743731498718 }, { "label": "completed_group_1", "mean": 0.08804173022508621, "median": 0.08880250155925751, "p05": 0.05664174258708954, "p95": 0.11779069900512695 }, { "label": "completed_group_2", "mean": 0.07975024729967117, "median": 0.07984748482704163, "p05": 0.05428915470838547, "p95": 0.10499341040849686 }, { "label": "completed_group_3", "mean": 0.09027000516653061, "median": 0.08898856490850449, "p05": 0.06189651042222977, "p95": 0.11996553838253021 }, { "label": "completed_group_4", "mean": 0.08430293202400208, "median": 0.0819794088602066, "p05": 0.05464346706867218, "p95": 0.12159879505634308 }, { "label": "completed_group_5", "mean": 0.07433462142944336, "median": 0.07287181913852692, "p05": 0.052520088851451874, "p95": 0.1010315865278244 }, { "label": "completed_group_6", "mean": 0.26602888107299805, "median": 0.2639757990837097, "p05": 0.15474572777748108, "p95": 0.3846370279788971 }, { "label": "completed_group_7", "mean": 0.20123262703418732, "median": 0.18140269815921783, "p05": 0.08419936150312424, "p95": 0.3782624900341034 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.7512003183364868, "group": 8, "latest_source_mass": 0.33482813835144043, "layer": 29, "max_source_mass": 0.33482813835144043, "mixer_index": 57, "normalized_entropy": 0.7970056117156376, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.07744461297988892, "median": 0.06970229744911194, "p05": 0.014401955530047417, "p95": 0.17144526541233063 }, { "label": "completed_group_1", "mean": 0.046086348593235016, "median": 0.04394831880927086, "p05": 0.028066720813512802, "p95": 0.0718703418970108 }, { "label": "completed_group_2", "mean": 0.05025433003902435, "median": 0.04861236736178398, "p05": 0.029644077643752098, "p95": 0.07761462777853012 }, { "label": "completed_group_3", "mean": 0.03748488053679466, "median": 0.03434944897890091, "p05": 0.020774798467755318, "p95": 0.05938909202814102 }, { "label": "completed_group_4", "mean": 0.04005061462521553, "median": 0.037498839199543, "p05": 0.019541850313544273, "p95": 0.06922487914562225 }, { "label": "completed_group_5", "mean": 0.041623204946517944, "median": 0.039552345871925354, "p05": 0.022081367671489716, "p95": 0.06821337342262268 }, { "label": "completed_group_6", "mean": 0.198423832654953, "median": 0.18318486213684082, "p05": 0.07548617571592331, "p95": 0.3721975088119507 }, { "label": "completed_group_7", "mean": 0.17380405962467194, "median": 0.1514919549226761, "p05": 0.05777408182621002, "p95": 0.36326974630355835 }, { "label": "current_group_partial", "mean": 0.33482813835144043, "median": 0.33602219820022583, "p05": 0.0795319601893425, "p95": 0.5979493856430054 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.0160694122314453, "group": 8, "latest_source_mass": 0.21711516380310059, "layer": 30, "max_source_mass": 0.21711516380310059, "mixer_index": 58, "normalized_entropy": 0.9175527313077866, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.09385271370410919, "median": 0.09211117774248123, "p05": 0.032377734780311584, "p95": 0.16496515274047852 }, { "label": "completed_group_1", "mean": 0.06208505108952522, "median": 0.06140740588307381, "p05": 0.0407925546169281, "p95": 0.0863042101264 }, { "label": "completed_group_2", "mean": 0.06560768187046051, "median": 0.06516212224960327, "p05": 0.041897669434547424, "p95": 0.09085705131292343 }, { "label": "completed_group_3", "mean": 0.07029052078723907, "median": 0.06780634075403214, "p05": 0.04846132919192314, "p95": 0.09917552024126053 }, { "label": "completed_group_4", "mean": 0.06935097277164459, "median": 0.06708167493343353, "p05": 0.04668639227747917, "p95": 0.09815650433301926 }, { "label": "completed_group_5", "mean": 0.056832700967788696, "median": 0.05595892667770386, "p05": 0.038822539150714874, "p95": 0.0771557092666626 }, { "label": "completed_group_6", "mean": 0.19425815343856812, "median": 0.19145764410495758, "p05": 0.12249615788459778, "p95": 0.27496030926704407 }, { "label": "completed_group_7", "mean": 0.17060703039169312, "median": 0.16317546367645264, "p05": 0.09239593893289566, "p95": 0.2726203501224518 }, { "label": "current_group_partial", "mean": 0.21711516380310059, "median": 0.21198657155036926, "p05": 0.11019522696733475, "p95": 0.3391069769859314 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 1.7837426662445068, "group": 8, "latest_source_mass": 0.29026859998703003, "layer": 30, "max_source_mass": 0.29026859998703003, "mixer_index": 59, "normalized_entropy": 0.8118162725118464, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.08836327493190765, "median": 0.07904411852359772, "p05": 0.02631576545536518, "p95": 0.17534467577934265 }, { "label": "completed_group_1", "mean": 0.04953558370471001, "median": 0.04658038914203644, "p05": 0.025181813165545464, "p95": 0.08318150043487549 }, { "label": "completed_group_2", "mean": 0.04813014343380928, "median": 0.045740917325019836, "p05": 0.026714935898780823, "p95": 0.07843323051929474 }, { "label": "completed_group_3", "mean": 0.0373263955116272, "median": 0.03401809185743332, "p05": 0.01990434154868126, "p95": 0.06290259957313538 }, { "label": "completed_group_4", "mean": 0.037015400826931, "median": 0.03405017405748367, "p05": 0.018646355718374252, "p95": 0.06544196605682373 }, { "label": "completed_group_5", "mean": 0.03390160948038101, "median": 0.03198609501123428, "p05": 0.018189728260040283, "p95": 0.05570102855563164 }, { "label": "completed_group_6", "mean": 0.1840541511774063, "median": 0.17327937483787537, "p05": 0.07966600358486176, "p95": 0.32982927560806274 }, { "label": "completed_group_7", "mean": 0.23140482604503632, "median": 0.22406929731369019, "p05": 0.08831030130386353, "p95": 0.40350067615509033 }, { "label": "current_group_partial", "mean": 0.29026859998703003, "median": 0.2745037376880646, "p05": 0.10280418395996094, "p95": 0.5399085879325867 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 1.9973840713500977, "group": 8, "latest_source_mass": 0.19101998209953308, "layer": 31, "max_source_mass": 0.20831164717674255, "mixer_index": 60, "normalized_entropy": 0.9090486661912383, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.10261696577072144, "median": 0.10389091819524765, "p05": 0.01371697522699833, "p95": 0.21163305640220642 }, { "label": "completed_group_1", "mean": 0.06039813905954361, "median": 0.06085003539919853, "p05": 0.026107093319296837, "p95": 0.09348798543214798 }, { "label": "completed_group_2", "mean": 0.06671924144029617, "median": 0.06656201928853989, "p05": 0.034684643149375916, "p95": 0.09819839894771576 }, { "label": "completed_group_3", "mean": 0.08377155661582947, "median": 0.0804532840847969, "p05": 0.05448959022760391, "p95": 0.12039553374052048 }, { "label": "completed_group_4", "mean": 0.07002593576908112, "median": 0.06737451255321503, "p05": 0.044841229915618896, "p95": 0.10655929893255234 }, { "label": "completed_group_5", "mean": 0.05768897384405136, "median": 0.056077782064676285, "p05": 0.039495617151260376, "p95": 0.08242568373680115 }, { "label": "completed_group_6", "mean": 0.20831164717674255, "median": 0.2021285891532898, "p05": 0.11652229726314545, "p95": 0.31706833839416504 }, { "label": "completed_group_7", "mean": 0.1594475358724594, "median": 0.1546279788017273, "p05": 0.06593266129493713, "p95": 0.2735573351383209 }, { "label": "current_group_partial", "mean": 0.19101998209953308, "median": 0.1713225245475769, "p05": 0.06408411264419556, "p95": 0.3850628137588501 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 1.969987154006958, "group": 8, "latest_source_mass": 0.16864418983459473, "layer": 31, "max_source_mass": 0.24503237009048462, "mixer_index": 61, "normalized_entropy": 0.8965797917640489, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.08120635151863098, "median": 0.061164144426584244, "p05": 0.0259944386780262, "p95": 0.23622018098831177 }, { "label": "completed_group_1", "mean": 0.07894632965326309, "median": 0.07658770680427551, "p05": 0.0496862456202507, "p95": 0.11596276611089706 }, { "label": "completed_group_2", "mean": 0.08615730702877045, "median": 0.08467075228691101, "p05": 0.050409119576215744, "p95": 0.12811316549777985 }, { "label": "completed_group_3", "mean": 0.07609779387712479, "median": 0.0722871869802475, "p05": 0.04099677503108978, "p95": 0.11950768530368805 }, { "label": "completed_group_4", "mean": 0.058728985488414764, "median": 0.056080836802721024, "p05": 0.03323862701654434, "p95": 0.09344407916069031 }, { "label": "completed_group_5", "mean": 0.05076971277594566, "median": 0.04946017265319824, "p05": 0.03047419898211956, "p95": 0.07563400268554688 }, { "label": "completed_group_6", "mean": 0.24503237009048462, "median": 0.24110376834869385, "p05": 0.10613486170768738, "p95": 0.3991503119468689 }, { "label": "completed_group_7", "mean": 0.1544169783592224, "median": 0.1400495171546936, "p05": 0.06199025735259056, "p95": 0.2995939254760742 }, { "label": "current_group_partial", "mean": 0.16864418983459473, "median": 0.15222924947738647, "p05": 0.0543956533074379, "p95": 0.3329208791255951 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.071648359298706, "group": 8, "latest_source_mass": 0.11520501971244812, "layer": 32, "max_source_mass": 0.20955611765384674, "mixer_index": 62, "normalized_entropy": 0.9428478002054053, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.10569609701633453, "median": 0.08689188957214355, "p05": 0.03173588588833809, "p95": 0.24571751058101654 }, { "label": "completed_group_1", "mean": 0.09922641515731812, "median": 0.09795533865690231, "p05": 0.06407676637172699, "p95": 0.14120742678642273 }, { "label": "completed_group_2", "mean": 0.08180510997772217, "median": 0.080217644572258, "p05": 0.054607540369033813, "p95": 0.11442367732524872 }, { "label": "completed_group_3", "mean": 0.10646799206733704, "median": 0.10510502755641937, "p05": 0.07415887713432312, "p95": 0.14216329157352448 }, { "label": "completed_group_4", "mean": 0.0826798751950264, "median": 0.08022409677505493, "p05": 0.05491747707128525, "p95": 0.11860857903957367 }, { "label": "completed_group_5", "mean": 0.06615597754716873, "median": 0.06485182791948318, "p05": 0.04477996751666069, "p95": 0.09238749742507935 }, { "label": "completed_group_6", "mean": 0.20955611765384674, "median": 0.206287682056427, "p05": 0.129981130361557, "p95": 0.2996031939983368 }, { "label": "completed_group_7", "mean": 0.13320739567279816, "median": 0.12465103715658188, "p05": 0.06303662061691284, "p95": 0.23065781593322754 }, { "label": "current_group_partial", "mean": 0.11520501971244812, "median": 0.10386130958795547, "p05": 0.03912060335278511, "p95": 0.22735844552516937 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 1.9781701564788818, "group": 8, "latest_source_mass": 0.16112206876277924, "layer": 32, "max_source_mass": 0.22251944243907928, "mixer_index": 63, "normalized_entropy": 0.9003040366848136, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.09446848928928375, "median": 0.074150450527668, "p05": 0.042592622339725494, "p95": 0.19443637132644653 }, { "label": "completed_group_1", "mean": 0.07056106626987457, "median": 0.06868506222963333, "p05": 0.04818308353424072, "p95": 0.0982733890414238 }, { "label": "completed_group_2", "mean": 0.08033448457717896, "median": 0.08047192543745041, "p05": 0.048296794295310974, "p95": 0.11253741383552551 }, { "label": "completed_group_3", "mean": 0.0658978745341301, "median": 0.06256154924631119, "p05": 0.04223886504769325, "p95": 0.0967884361743927 }, { "label": "completed_group_4", "mean": 0.058234501630067825, "median": 0.0563921295106411, "p05": 0.03529864549636841, "p95": 0.08846980333328247 }, { "label": "completed_group_5", "mean": 0.04316519945859909, "median": 0.042268507182598114, "p05": 0.029443470761179924, "p95": 0.05996086448431015 }, { "label": "completed_group_6", "mean": 0.2036968618631363, "median": 0.19645483791828156, "p05": 0.09398645162582397, "p95": 0.3395216464996338 }, { "label": "completed_group_7", "mean": 0.22251944243907928, "median": 0.2130982130765915, "p05": 0.11867167800664902, "p95": 0.35465875267982483 }, { "label": "current_group_partial", "mean": 0.16112206876277924, "median": 0.14023703336715698, "p05": 0.06008770316839218, "p95": 0.3247341811656952 } ], "sources": 9 }, { "branch": "output", "entropy_mean": 1.6036605834960938, "group": null, "latest_source_mass": 0.31639498472213745, "layer": null, "max_source_mass": 0.3412485122680664, "mixer_index": null, "normalized_entropy": 0.7298573846467136, "offset": null, "source_summaries": [ { "label": "embedding", "mean": 0.09759747982025146, "median": 0.09915056824684143, "p05": 0.025461740791797638, "p95": 0.16398483514785767 }, { "label": "completed_group_1", "mean": 0.04215940088033676, "median": 0.04050712287425995, "p05": 0.025720715522766113, "p95": 0.06398092210292816 }, { "label": "completed_group_2", "mean": 0.041145481169223785, "median": 0.03881259262561798, "p05": 0.023510465398430824, "p95": 0.0662468671798706 }, { "label": "completed_group_3", "mean": 0.029554028064012527, "median": 0.027420252561569214, "p05": 0.01627083122730255, "p95": 0.04924973472952843 }, { "label": "completed_group_4", "mean": 0.0289996936917305, "median": 0.027226522564888, "p05": 0.01560397818684578, "p95": 0.048351090401411057 }, { "label": "completed_group_5", "mean": 0.03647579252719879, "median": 0.0347352996468544, "p05": 0.020744025707244873, "p95": 0.058257561177015305 }, { "label": "completed_group_6", "mean": 0.06642461568117142, "median": 0.057809896767139435, "p05": 0.022837381809949875, "p95": 0.14078694581985474 }, { "label": "completed_group_7", "mean": 0.3412485122680664, "median": 0.3336990475654602, "p05": 0.0819513276219368, "p95": 0.6423859596252441 }, { "label": "completed_group_8", "mean": 0.31639498472213745, "median": 0.30713334679603577, "p05": 0.05841828137636185, "p95": 0.6156357526779175 } ], "sources": 9 } ], "mode": "uniform_except_group_7", "optimizer_state_after": "ed03e6fbd4a12d8b063dcb22e0437754285f54d585374cd52fbd534f05d24637", "optimizer_state_before": "ed03e6fbd4a12d8b063dcb22e0437754285f54d585374cd52fbd534f05d24637", "optimizer_state_unchanged": true, "positions": { "attention_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "5e6ecbcd78543fc42d4098a51169dca446d404ce384015643e84d2c78433c949", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.0933690071105957, 0.24537226557731628, 0.07698935270309448, 0.0523640513420105, 0.09380833059549332, 0.03647654876112938, 0.039120711386203766, 0.06371714919805527, 0.05156813934445381, 0.03728010877966881, 0.03983496502041817, 0.040284499526023865, 0.046871740370988846, 0.04314620420336723, 0.04033593460917473, 0.04471700266003609, 0.08397205173969269, 0.0528092160820961, 0.04443390294909477, 0.07560808956623077, 0.03708760067820549, 0.03599191829562187, 0.033304885029792786, 0.04061548039317131, 0.025487789884209633, 0.021565141156315804, 0.02342291921377182, 0.017244853079319, 0.024166326969861984, 0.016819356009364128, 0.021590016782283783, 0.026201345026493073 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 8.806625544366398e-06, "first_to_last_ratio": 1.5630486193480198, "imbalance_abs_log_ratio": 0.4466381573842201, "last_quartile_mean": 5.634262066678275e-06, "mean": 6.495427548713906e-06, "normalized": [ 1.5749936942252394, 1.7035207101025074, 1.4734702419933616, 1.320317915078982, 1.308280190322649, 1.2336879110032513, 1.1651301763871378, 1.0671529599329206, 0.9989252542480824, 0.9181142329156451, 0.8700423055989427, 0.8307143246911584, 0.8187374389550371, 0.7858142876503071, 0.7643928616327025, 0.7403476566041925, 0.7464427602861687, 0.7265514111232573, 0.7303447135319046, 0.7313012653537153, 1.1662258388339897, 1.144315600343291, 1.1279342915915853, 1.1138840484520443, 1.6760351950012624, 1.4686703303412103, 1.1046276260615955, 0.7050286100754264, 0.5469755915668992, 0.5132200868544617, 0.47233384991403804, 0.45246661932703336 ], "peak_layer": 2, "peak_normalized": 1.7035207101025074, "population_cv": 0.3420749927540303, "rest_mean": 6.199910861444853e-06, "spike_contrast": 1.305053869448887, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 8.091217659966788e-06, "top_five_layers": [ 2, 25, 1, 3, 26 ], "top_five_spike_overlap": 1 }, "values": [ 1.0230257430521306e-05, 1.1065095350204501e-05, 9.570819202053826e-06, 8.576029358664528e-06, 8.497839189658407e-06, 8.013330443645827e-06, 7.568018645542907e-06, 6.931614734639879e-06, 6.488446615549037e-06, 5.9635444813466165e-06, 5.651296760333935e-06, 5.395844709710218e-06, 5.318049716152018e-06, 5.1041997721767984e-06, 4.9650584514893126e-06, 4.808874564332655e-06, 4.84846486870083e-06, 4.719262051366968e-06, 4.7439011723326985e-06, 4.75011438538786e-06, 7.57513544158428e-06, 7.432819074892905e-06, 7.326415470743086e-06, 7.2351531343883835e-06, 1.0886565178225283e-05, 9.53964172367705e-06, 7.17502871339093e-06, 4.579462256515399e-06, 3.552840325937723e-06, 3.333583890707814e-06, 3.068010300921742e-06, 2.93896414405026e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 3.504698725009803e-05, "first_to_last_ratio": 1.555874353759609, "imbalance_abs_log_ratio": 0.44203767298396307, "last_quartile_mean": 2.25255896566523e-05, "mean": 2.5910327906331077e-05, "normalized": [ 1.5944025925413352, 1.724762318043881, 1.481618533189431, 1.312420821558483, 1.2902024736297426, 1.2106325579112314, 1.148708652006666, 1.0582613032744517, 0.993291980090295, 0.915967936759783, 0.8686802164407651, 0.8316469454346875, 0.8208911044617463, 0.7899785746022444, 0.7686829080775075, 0.7448322642251208, 0.7530924496224007, 0.7287633705749837, 0.7302889575483078, 0.7266964462885446, 1.16508826527136, 1.1455298485391718, 1.1282696056212924, 1.1123521403960532, 1.677536727607123, 1.4677251682639885, 1.1027401912892636, 0.7056687455949787, 0.5505997202062364, 0.5173209193733388, 0.4766714084067027, 0.4566748531488823 ], "peak_layer": 2, "peak_normalized": 1.724762318043881, "population_cv": 0.3422469930502032, "rest_mean": 2.4731142561736136e-05, "spike_contrast": 1.3051531560488414, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.227792876714375e-05, "top_five_layers": [ 2, 25, 1, 3, 26 ], "top_five_spike_overlap": 1 }, "values": [ 4.1311493987450376e-05, 4.468915722100064e-05, 3.838922202703543e-05, 3.400525383767672e-05, 3.3429569157306105e-05, 3.136788654956035e-05, 2.9763417842332274e-05, 2.741989737842232e-05, 2.573652091086842e-05, 2.3733029593131505e-05, 2.2507789253722876e-05, 2.1548245058511384e-05, 2.1269557691994123e-05, 2.046860390692018e-05, 1.991682620428037e-05, 1.929884820128791e-05, 1.951287231349852e-05, 1.8882497897720896e-05, 1.892202635644935e-05, 1.88289432117017e-05, 3.0187818992999382e-05, 2.9681054002139717e-05, 2.923383544839453e-05, 2.8821408704970963e-05, 4.346552668721415e-05, 3.80292403860949e-05, 2.857235995179508e-05, 1.8284108591615222e-05, 1.4266219295677729e-05, 1.3403954653767869e-05, 1.2350712495390326e-05, 1.1832595191663131e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.03108001803047955, "first_to_last_ratio": 1.5558743482578572, "imbalance_abs_log_ratio": 0.44203766944784734, "last_quartile_mean": 0.019975917763076723, "mean": 0.022977537679253146, "normalized": [ 1.5944025021144241, 1.724762400055327, 1.4816185201099057, 1.3124208220207036, 1.290202466735928, 1.2106324878901569, 1.1487086261799482, 1.0582612714574344, 0.9932920071482564, 0.9159678992299886, 0.8686802579746635, 0.8316469366874443, 0.8208911564212863, 0.7899785528522092, 0.7686829461242921, 0.7448322913630246, 0.753092443317538, 0.7287633503604891, 0.7302889699546278, 0.7266964681717669, 1.1650882844993984, 1.1455298737280375, 1.1282696199967315, 1.1123521871249848, 1.6775366104339196, 1.4677251677719554, 1.102740216235729, 0.7056687771058386, 0.5505997188610687, 0.5173209052418795, 0.4766714115596055, 0.4566748512714372 ], "peak_layer": 2, "peak_normalized": 1.724762400055327, "population_cv": 0.34224698279885796, "rest_mean": 0.02193182434334799, "spike_contrast": 1.3051531530172442, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.028624389693140982, "top_five_layers": [ 2, 25, 1, 3, 26 ], "top_five_spike_overlap": 1 }, "values": [ 0.036635443568229675, 0.039630793035030365, 0.034043945372104645, 0.030156198889017105, 0.029645675793290138, 0.02781735360622406, 0.026394495740532875, 0.02431623823940754, 0.022823404520750046, 0.02104668691754341, 0.019960133358836174, 0.019109198823571205, 0.018862057477235794, 0.01815176196396351, 0.017662441357970238, 0.017114412039518356, 0.017304209992289543, 0.0167451873421669, 0.016780242323875427, 0.01669769547879696, 0.026770859956741333, 0.02632145583629608, 0.025924857705831528, 0.025559114292263985, 0.03854566067457199, 0.03372471034526825, 0.025338254868984222, 0.016214530915021896, 0.012651425786316395, 0.011886760592460632, 0.01095273531973362, 0.010493263602256775 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 1.3540603163164633e-06, "first_to_last_ratio": 0.895410858285723, "imbalance_abs_log_ratio": 0.1104726065057547, "last_quartile_mean": 1.5122223544494773e-06, "mean": 1.3040891211346661e-06, "normalized": [ 2.1107437739372075, 1.1248879449474893, 1.0256937169934692, 0.9007476929609765, 0.811366043426423, 0.7837986078100881, 0.7743838189527891, 0.7749291995140253, 0.7513809815120179, 0.7213259445026405, 0.7107658215958655, 0.7118905382376156, 0.6961550101056764, 0.7013294127930443, 0.7136764067617851, 0.7164990301332214, 0.7074866948182731, 0.7313758192596528, 0.7600357595424537, 0.7962397531544899, 1.3487320559866687, 1.4055539803876635, 1.4577703333292424, 1.4864279198276518, 2.2137579324270726, 1.9874155843764891, 1.5422504448114454, 0.9473015380741447, 0.7099700245285311, 0.6750358540071224, 0.6196376204542011, 0.5814347408305626 ], "peak_layer": 25, "peak_normalized": 2.2137579324270726, "population_cv": 0.44502244442071587, "rest_mean": 1.1634289952174488e-06, "spike_contrast": 1.7737685837045307, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.0636538010876394e-06, "top_five_layers": [ 25, 1, 26, 27, 24 ], "top_five_spike_overlap": 2 }, "values": [ 2.7525979930942412e-06, 1.4669541315015522e-06, 1.3375960179473623e-06, 1.1746552672775579e-06, 1.0580936304904753e-06, 1.0221432376056327e-06, 1.009865513879049e-06, 1.0105767387358355e-06, 9.798677638173103e-07, 9.406733170180814e-07, 9.269019756175112e-07, 9.283687063543766e-07, 9.078481753022061e-07, 9.145960575551726e-07, 9.306976380685228e-07, 9.343785905002733e-07, 9.226257020600315e-07, 9.53779249357467e-07, 9.91154365692637e-07, 1.0383675999037223e-06, 1.7588668015378062e-06, 1.83296765499108e-06, 1.901062432807521e-06, 1.9384344795980724e-06, 2.886937636503717e-06, 2.5917670427588746e-06, 2.0112320271437056e-06, 1.235365630236629e-06, 9.258641853193694e-07, 8.803069135865371e-07, 8.08062679880095e-07, 7.582427201668906e-07 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 2.6826776775124017e-05, "first_to_last_ratio": 1.7808426615871156, "imbalance_abs_log_ratio": 0.5770866576690369, "last_quartile_mean": 1.506409148532839e-05, "mean": 1.8887763218344844e-05, "normalized": [ 1.6688978585157723, 1.7802306893938922, 1.5513427500707169, 1.393336962824707, 1.3714993784350937, 1.2838699484700635, 1.2047806691021654, 1.1086478771180956, 1.0355097762403505, 0.9557614805376662, 0.9047104907663882, 0.8641109241598542, 0.8541535465595488, 0.816883635185939, 0.7910369625308257, 0.7641276103862781, 0.7785007248672575, 0.746430330067535, 0.7506111304658604, 0.7465761361074102, 1.1083785115345217, 1.0754902931610733, 1.0469267551222607, 1.017719513272004, 1.571873858412148, 1.308955810216977, 0.9748786362519846, 0.6421390449077502, 0.5159694382679512, 0.4848134611357421, 0.45009669634280136, 0.4317390995693655 ], "peak_layer": 2, "peak_normalized": 1.7802306893938922, "population_cv": 0.3494649075198096, "rest_mean": 1.8313862778153702e-05, "spike_contrast": 1.2005564233889927, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.1986825595377012e-05, "top_five_layers": [ 2, 1, 25, 3, 4 ], "top_five_spike_overlap": 1 }, "values": [ 3.152174758724868e-05, 3.362457573530264e-05, 2.9301394533831626e-05, 2.6317018637200817e-05, 2.590455551398918e-05, 2.4249431589851156e-05, 2.275561200804077e-05, 2.093987859552726e-05, 1.9558463463908993e-05, 1.8052196537610143e-05, 1.70879575307481e-05, 1.6321122529916465e-05, 1.6133049939526245e-05, 1.5429104678332806e-05, 1.4940918845240958e-05, 1.4432661373575684e-05, 1.4704137356602587e-05, 1.4098399333306588e-05, 1.4177365301293321e-05, 1.4101153283263557e-05, 2.0934790882165544e-05, 2.0313606000854634e-05, 1.9774104657699354e-05, 1.9222445189370774e-05, 2.968918124679476e-05, 2.472324740665499e-05, 1.8413276848150417e-05, 1.2128570233471692e-05, 9.745508577907458e-06, 9.157041858998127e-06, 8.501319825882092e-06, 8.154585884767585e-06 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 1.9906377247025375e-05, "first_to_last_ratio": 1.8413746446920487, "imbalance_abs_log_ratio": 0.6105123821949934, "last_quartile_mean": 1.0810606795530475e-05, "mean": 1.3778397345731719e-05, "normalized": [ 1.6664505437252657, 1.7912816595804526, 1.5891726550469536, 1.4374596102552084, 1.43006280243105, 1.316241813041211, 1.2083826013831032, 1.1189701932958183, 1.0465934538522217, 0.9653525322075278, 0.9221076957451294, 0.875907779835731, 0.8687089982432784, 0.8294489116240055, 0.807940424484781, 0.7776193079734464, 0.8072240975959576, 0.772438149647099, 0.7836992413567619, 0.7850500442919703, 1.0465469836965149, 0.9997968188979502, 0.9597722190068056, 0.916927197506221, 1.494523509270956, 1.1925937058942337, 0.9014934290595438, 0.6362105581485062, 0.5473760236097105, 0.5208516970432409, 0.49712346786408346, 0.4866718743852613 ], "peak_layer": 2, "peak_normalized": 1.7912816595804526, "population_cv": 0.33893111943396753, "rest_mean": 1.3565308445659294e-05, "spike_contrast": 1.100533575475013, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.4929077406122814e-05, "top_five_layers": [ 2, 1, 3, 25, 4 ], "top_five_spike_overlap": 1 }, "values": [ 2.296101774845738e-05, 2.4680990463821217e-05, 2.1896252292208374e-05, 1.9805889678536914e-05, 1.9703973521245643e-05, 1.8135702703148127e-05, 1.6649575627525337e-05, 1.541761594126001e-05, 1.4420380466617644e-05, 1.3301010767463595e-05, 1.2705166227533482e-05, 1.2068605428794399e-05, 1.1969417755608447e-05, 1.142847668234026e-05, 1.1132124200230464e-05, 1.071434780897107e-05, 1.1122254363726825e-05, 1.0642959750839509e-05, 1.079811954696197e-05, 1.0816731446539052e-05, 1.4419740182347596e-05, 1.3775597835774533e-05, 1.3224122994870413e-05, 1.2633787264348939e-05, 2.0592138753272593e-05, 1.6432029951829463e-05, 1.2421134670148604e-05, 8.765961865719873e-06, 7.5419643508212175e-06, 7.176501640060451e-06, 6.849564670119435e-06, 6.705558462272165e-06 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 7.224147339002229e-05, "first_to_last_ratio": 1.5165661976092093, "imbalance_abs_log_ratio": 0.4164486987659755, "last_quartile_mean": 4.763489619108441e-05, "mean": 5.4422049345248524e-05, "normalized": [ 1.5779000491493227, 1.6582291046256656, 1.4285647041291294, 1.2842276815653029, 1.2647281274920252, 1.1984005516039755, 1.1496093519432375, 1.0577822563478, 0.9907014426509673, 0.9263268837864721, 0.8841305258035396, 0.8398816148459275, 0.8329922430132144, 0.7989089006409306, 0.7778778031316362, 0.7483117484915442, 0.7515903517547307, 0.7262582182270726, 0.7235115206476356, 0.7212055487403063, 1.1901150694023914, 1.1640410621436472, 1.1556673412211194, 1.1467442674132011, 1.6643277371931944, 1.4767758010471783, 1.1318629408210734, 0.70793137012464, 0.5620287118917052, 0.5220555017439436, 0.4823081359285368, 0.45500343247893327 ], "peak_layer": 25, "peak_normalized": 1.6643277371931944, "population_cv": 0.33301148613365084, "rest_mean": 5.175961086933967e-05, "spike_contrast": 1.329206614184773, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.879921711515636e-05, "top_five_layers": [ 25, 2, 1, 26, 3 ], "top_five_spike_overlap": 1 }, "values": [ 8.587255433667451e-05, 9.024422615766525e-05, 7.774541882099584e-05, 6.989030225668103e-05, 6.882909656269476e-05, 6.52194139547646e-05, 6.256409687921405e-05, 5.7566678151488304e-05, 5.391600279835984e-05, 5.041260737925768e-05, 4.8116195102920756e-05, 4.570807868731208e-05, 4.53331449534744e-05, 4.347825961303897e-05, 4.233370418660343e-05, 4.0724658902036026e-05, 4.0903087210608646e-05, 3.952446058974601e-05, 3.937497967854142e-05, 3.9249483961611986e-05, 6.476850103354082e-05, 6.334950012387708e-05, 6.289378507062793e-05, 6.24081731075421e-05, 9.057612624019384e-05, 8.036916551645845e-05, 6.159830081742257e-05, 3.8527075957972556e-05, 3.058675429201685e-05, 2.8411330276867375e-05, 2.624819717311766e-05, 2.476221925462596e-05 ] } } }, "mlp_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "c0b8018c349b1bd09e5d17858eb312a2323da0e92de9651e2bb08ae4adf115c7", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.13583384454250336, 0.0415157787501812, 0.018140379339456558, 0.039518751204013824, 0.022877270355820656, 0.015532293356955051, 0.02314678393304348, 0.05432315170764923, 0.03743875026702881, 0.04006272554397583, 0.045583780854940414, 0.05995568633079529, 0.038651175796985626, 0.04559725895524025, 0.05632574483752251, 0.07952121645212173, 0.042337141931056976, 0.05044656619429588, 0.04418695345520973, 0.06310147047042847, 0.029010163620114326, 0.04586305841803551, 0.06033914536237717, 0.07656749337911606, 0.045884523540735245, 0.051604680716991425, 0.06502768397331238, 0.06792232394218445, 0.060306549072265625, 0.08025261014699936, 0.09663844108581543, 0.12230555713176727 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 8.59067426972615e-06, "first_to_last_ratio": 1.8480553924768979, "imbalance_abs_log_ratio": 0.6141339470543082, "last_quartile_mean": 4.648493927561503e-06, "mean": 6.141739163467719e-06, "normalized": [ 1.5093943986289327, 1.7774339591082804, 1.546607520838829, 1.35876223353886, 1.3760658755649344, 1.2923032167466824, 1.2091130515533581, 1.120211129607037, 1.0049326671196686, 0.9364260897087959, 0.8893551485341262, 0.8522976628477164, 0.8489944220188363, 0.8144483707949101, 0.7899467987049812, 0.769071365102793, 0.7833374294047779, 0.7615634207079022, 0.7677616341494372, 0.7793216072714094, 1.2223146123833326, 1.1953514361735968, 1.1812873587584203, 1.158744052497784, 1.566604074548636, 1.1748023061675101, 0.7516023129941929, 0.5787777272214111, 0.5425954933637996, 0.5001050194833493, 0.47913592260703614, 0.46133168184866274 ], "peak_layer": 2, "peak_normalized": 1.7774339591082804, "population_cv": 0.3415319455327771, "rest_mean": 5.840497882066605e-06, "spike_contrast": 1.3300992894607375, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 7.768442083033733e-06, "top_five_layers": [ 2, 25, 3, 1, 5 ], "top_five_spike_overlap": 1 }, "values": [ 9.27030669117812e-06, 1.0916535757132806e-05, 9.498859981249552e-06, 8.345163223566487e-06, 8.451437679468654e-06, 7.936989277368411e-06, 7.426056981785223e-06, 6.880044566059951e-06, 6.172034318296937e-06, 5.751284788857447e-06, 5.462187345983693e-06, 5.234589934843825e-06, 5.214302291278727e-06, 5.002129455533577e-06, 4.851647190662334e-06, 4.723435722553404e-06, 4.8110541683854535e-06, 4.677323886426166e-06, 4.715391696663573e-06, 4.786390036315424e-06, 7.507137524953578e-06, 7.341536729654763e-06, 7.255158834595932e-06, 7.1167037276609335e-06, 9.62167359830346e-06, 7.21532933312119e-06, 4.616145361069357e-06, 3.5547018342185766e-06, 3.3324799915135372e-06, 3.071514584007673e-06, 2.942727860499872e-06, 2.833378857758362e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 3.4157302707171766e-05, "first_to_last_ratio": 1.836603963952192, "imbalance_abs_log_ratio": 0.6079181945049807, "last_quartile_mean": 1.859807741766417e-05, "mean": 2.449506737889351e-05, "normalized": [ 1.5280925929985913, 1.7998236884207635, 1.5548656373619265, 1.3487219435234745, 1.355966246612284, 1.2670212882543248, 1.1917350128384379, 1.1094245392433835, 1.000277009668222, 0.935090163375531, 0.8890480629161511, 0.854396020233672, 0.8521902185346131, 0.8202385452631418, 0.7954405043332893, 0.7741114920316194, 0.7900785289265344, 0.7637913639234517, 0.7672638827441962, 0.7737125704477471, 1.220849530854245, 1.1957716804441187, 1.1801870052832615, 1.157837965640551, 1.5666154077509415, 1.1729654998352281, 0.7501430772996489, 0.5827291220768122, 0.5468132955195877, 0.5045610986258278, 0.4838352213026038, 0.46640178371581853 ], "peak_layer": 2, "peak_normalized": 1.7998236884207635, "population_cv": 0.34089985587125604, "rest_mean": 2.3296386205873676e-05, "spike_contrast": 1.3293025553204776, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.0967945713200605e-05, "top_five_layers": [ 2, 25, 3, 1, 5 ], "top_five_spike_overlap": 1 }, "values": [ 3.743073102668859e-05, 4.408680251799524e-05, 3.808653855230659e-05, 3.3037034881999716e-05, 3.321448457427323e-05, 3.1035771826282144e-05, 2.9191629437264055e-05, 2.7175428840564564e-05, 2.4501852749381214e-05, 2.290509655722417e-05, 2.1777292204205878e-05, 2.0928488083882257e-05, 2.087445682263933e-05, 2.0091798432986252e-05, 1.9484368749544956e-05, 1.8961913156090304e-05, 1.9353026800672524e-05, 1.8709120922721922e-05, 1.8794180505210534e-05, 1.8952141545014456e-05, 2.990479151776526e-05, 2.9290507882251404e-05, 2.8908760214108042e-05, 2.8361318982206285e-05, 3.837434996967204e-05, 2.873186895158142e-05, 1.8374805222265422e-05, 1.4273989108914975e-05, 1.339422851742711e-05, 1.2359258107608184e-05, 1.1851576346089132e-05, 1.1424543117755093e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.03029103809967637, "first_to_last_ratio": 1.8366040001170014, "imbalance_abs_log_ratio": 0.6079182141961116, "last_quartile_mean": 0.01649296097457409, "mean": 0.021722470642998815, "normalized": [ 1.5280925650785975, 1.7998236412005102, 1.5548658154644794, 1.3487219742830503, 1.3559663424775814, 1.2670211756715235, 1.1917350535145705, 1.1094245909643194, 1.000276996223074, 0.9350901158437585, 0.8890480550986143, 0.8543959915839366, 0.8521902257386845, 0.8202385190113913, 0.7954404597406306, 0.7741114826033392, 0.7900785039858763, 0.7637913555505749, 0.7672638675796205, 0.7737125857477913, 1.2208494648735873, 1.1957716976185624, 1.1801870238454164, 1.1578379957636717, 1.566615437246143, 1.172965464456469, 0.7501431150725357, 0.582729140256694, 0.5468132888431574, 0.5045610912152754, 0.483835218341418, 0.4664017451051451 ], "peak_layer": 2, "peak_normalized": 1.7998236412005102, "population_cv": 0.34089986725223026, "rest_mean": 0.02065946817122124, "spike_contrast": 1.3293025630182196, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.027462683990597724, "top_five_layers": [ 2, 25, 3, 1, 5 ], "top_five_spike_overlap": 1 }, "values": [ 0.03319394588470459, 0.039096616208553314, 0.03377552703022957, 0.02929757349193096, 0.02945493906736374, 0.027522830292582512, 0.02588742971420288, 0.0240994431078434, 0.02172848768532276, 0.020312467589974403, 0.01931232027709484, 0.018559591844677925, 0.018511677160859108, 0.017817607149481773, 0.01727893203496933, 0.016815613955259323, 0.01716245710849762, 0.01659143529832363, 0.016666866838932037, 0.0168069489300251, 0.026519866660237312, 0.025975115597248077, 0.0256365779787302, 0.025151101872324944, 0.03403075784444809, 0.02547970786690712, 0.01629496179521084, 0.012658316642045975, 0.011878135614097118, 0.010960313491523266, 0.010510096326470375, 0.01013139821588993 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 1.290221064209618e-06, "first_to_last_ratio": 1.029023239933773, "imbalance_abs_log_ratio": 0.028610041566529118, "last_quartile_mean": 1.253830831160485e-06, "mean": 1.2122229975375376e-06, "normalized": [ 2.0555056873406334, 1.1658252426273945, 1.0706033027594348, 0.9039312630645182, 0.8621069859409929, 0.8310237635167318, 0.8183958732329527, 0.8073518969700927, 0.7752609210902222, 0.7516483727754637, 0.7424033567085462, 0.7364337855547445, 0.7333130838552302, 0.7355320546463936, 0.7447817599016021, 0.7514838760502169, 0.7526180969140335, 0.7784317971044497, 0.8099600234975225, 0.8545208171238841, 1.4396228204506287, 1.492913006999434, 1.5515972606230923, 1.5601463073638662, 2.147231649770154, 1.6657750565897105, 1.0896777953218548, 0.7693749112717653, 0.7163115407605933, 0.6628171869535997, 0.6315327983516565, 0.5918677048685849 ], "peak_layer": 25, "peak_normalized": 2.147231649770154, "population_cv": 0.4080594007327774, "rest_mean": 1.0689332536154932e-06, "spike_contrast": 1.857915457302219, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.985987614716578e-06, "top_five_layers": [ 25, 1, 26, 24, 23 ], "top_five_spike_overlap": 3 }, "values": [ 2.4917312657635193e-06, 1.413240170222707e-06, 1.29780994484463e-06, 1.0957662652799627e-06, 1.0450659146954422e-06, 1.0073861176351784e-06, 9.920782986228005e-07, 9.78690536612703e-07, 9.397891176377016e-07, 9.111654435400851e-07, 8.999584224511636e-07, 8.927219710130885e-07, 8.889389846444828e-07, 8.916288720683951e-07, 9.028415774992027e-07, 9.109660368267214e-07, 9.123409654421266e-07, 9.436329264644883e-07, 9.818521675697411e-07, 1.0358697863921407e-06, 1.7451438907301053e-06, 1.8097434804076329e-06, 1.880881882243557e-06, 1.8912452333097463e-06, 2.6029235868918477e-06, 2.01929083232244e-06, 1.3209324833951541e-06, 9.326539611720364e-07, 8.683293231115385e-07, 8.03482237188291e-07, 7.655585818611144e-07, 7.174756433414586e-07 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 2.6237477868562564e-05, "first_to_last_ratio": 2.0853511639588183, "imbalance_abs_log_ratio": 0.7349372650301057, "last_quartile_mean": 1.2581803161992866e-05, "mean": 1.7980487513113985e-05, "normalized": [ 1.6149798441404883, 1.8440811141623856, 1.6189665399607576, 1.4287691408443297, 1.432778699871445, 1.3360673349135428, 1.2443709030961805, 1.1537423651348564, 1.03964484651091, 0.9720376357153375, 0.9222845661737246, 0.8842553646770565, 0.882085484483975, 0.8438490472385294, 0.8142901074727685, 0.7901798442075099, 0.8119824410508196, 0.7786992771026724, 0.7846060776394639, 0.7927091117559889, 1.1553038412307217, 1.1157675902102373, 1.087164504470507, 1.0534034388005025, 1.4315135349828871, 1.0505752803465984, 0.686964807383655, 0.5438937381373701, 0.5099816336716981, 0.47546653741662515, 0.4573641887379033, 0.4422211584585528 ], "peak_layer": 2, "peak_normalized": 1.8440811141623856, "population_cv": 0.35430775598135755, "rest_mean": 1.7418994907024575e-05, "spike_contrast": 1.206300805422651, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.1012547585996798e-05, "top_five_layers": [ 2, 3, 1, 5, 25 ], "top_five_spike_overlap": 1 }, "values": [ 2.903812492149882e-05, 3.31574774463661e-05, 2.9109807655913755e-05, 2.568996569607407e-05, 2.5762059522094205e-05, 2.4023142032092437e-05, 2.2374395484803244e-05, 2.0744850189657882e-05, 1.8693321180762723e-05, 1.7477710571256466e-05, 1.6583126125624403e-05, 1.5899342542979866e-05, 1.586032703926321e-05, 1.517281725682551e-05, 1.4641333109466359e-05, 1.4207818821887486e-05, 1.4599840142182074e-05, 1.4001392628415488e-05, 1.4107599781709723e-05, 1.4253296285460237e-05, 2.0772926291101612e-05, 2.0062045223312452e-05, 1.9547747797332704e-05, 1.8940707377623767e-05, 2.573931124061346e-05, 1.888985570985824e-05, 1.2351962141110562e-05, 9.77947456703987e-06, 9.169718396151438e-06, 8.549120138923172e-06, 8.223631084547378e-06, 7.95135201769881e-06 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 1.9581357946663047e-05, "first_to_last_ratio": 2.1147071479688826, "imbalance_abs_log_ratio": 0.7489163385938595, "last_quartile_mean": 9.259607395506464e-06, "mean": 1.3228497834916197e-05, "normalized": [ 1.649838715876943, 1.8401311508969034, 1.6493718851678099, 1.4735245260904504, 1.4773895268481263, 1.3591211636889844, 1.2348501413909643, 1.1576967069549504, 1.04802016017699, 0.9826541667702433, 0.9365814821655866, 0.8926177860738392, 0.8930047949194518, 0.8537331255942407, 0.825930503629078, 0.8008027735003702, 0.8347121461645919, 0.7977482293552509, 0.8145627977219401, 0.8329343393197081, 1.0852645191752197, 1.0299790439677483, 0.9879850434556696, 0.9417515463283823, 1.3135203974919107, 0.9690640991779779, 0.6728211689679013, 0.5718069100699238, 0.5432438146918422, 0.5178209980530315, 0.509136778067568, 0.5023795582464018 ], "peak_layer": 2, "peak_normalized": 1.8401311508969034, "population_cv": 0.34430481386691286, "rest_mean": 1.3052852510655714e-05, "spike_contrast": 1.0861214109597435, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.4176982585922814e-05, "top_five_layers": [ 2, 1, 3, 5, 4 ], "top_five_spike_overlap": 0 }, "values": [ 2.182488788093906e-05, 2.4342170945601538e-05, 2.181871241191402e-05, 1.9492516003083438e-05, 1.9543644157238305e-05, 1.7979131371248513e-05, 1.633521242183633e-05, 1.5314588381443173e-05, 1.3863732419849839e-05, 1.2999038517591543e-05, 1.2389566109050065e-05, 1.1807992450485472e-05, 1.1813111996161751e-05, 1.1293606803519651e-05, 1.0925819879048504e-05, 1.0593417755444534e-05, 1.1041987818316557e-05, 1.0553010724834166e-05, 1.0775442206067964e-05, 1.1018470104318112e-05, 1.4356419342220761e-05, 1.3625075553136412e-05, 1.3069558008282911e-05, 1.2457958291633986e-05, 1.7375901734340005e-05, 1.2819262337870896e-05, 8.900413376977667e-06, 7.564146471850108e-06, 7.186299626482651e-06, 6.849993951618671e-06, 6.735114766343031e-06, 6.645726898568682e-06 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 7.04306489751616e-05, "first_to_last_ratio": 1.7947172331622148, "imbalance_abs_log_ratio": 0.584847479259299, "last_quartile_mean": 3.924331235793943e-05, "mean": 5.1402732481165e-05, "normalized": [ 1.5089267805038495, 1.737582401614998, 1.5066407791611753, 1.327495700690226, 1.3265500180295196, 1.2567212406691055, 1.193437646100348, 1.1040314258480088, 0.9959848109321353, 0.9451149113944519, 0.9002356093072116, 0.8682420650670096, 0.8596123038373616, 0.8309187405446787, 0.8042062840502662, 0.7758706250544842, 0.7877393600131564, 0.7578876504055657, 0.7609530861998457, 0.7692211921087176, 1.2480333031823778, 1.2188913863755821, 1.2106830137525684, 1.1974358129068603, 1.5569642323717137, 1.1990770627872507, 0.7506025250740437, 0.5961931135153943, 0.5532039382039474, 0.5111675937307798, 0.48181222243328736, 0.45856316413407916 ], "peak_layer": 2, "peak_normalized": 1.737582401614998, "population_cv": 0.3324715922220501, "rest_mean": 4.8676469102954594e-05, "spike_contrast": 1.3584501082780371, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.61245547235012e-05, "top_five_layers": [ 2, 25, 1, 3, 4 ], "top_five_spike_overlap": 1 }, "values": [ 7.756295963190496e-05, 8.931648335419595e-05, 7.74454529164359e-05, 6.823690637247637e-05, 6.818829569965601e-05, 6.459890573751181e-05, 6.134595605544746e-05, 5.675023203366436e-05, 5.1196340791648254e-05, 4.858148895436898e-05, 4.6274570195237175e-05, 4.463001459953375e-05, 4.418642129166983e-05, 4.271149373380467e-05, 4.1338400478707626e-05, 3.988187017966993e-05, 4.0491955587640405e-05, 3.8957496144576e-05, 3.911506792064756e-05, 3.9540071156807244e-05, 6.415232201106846e-05, 6.265434785746038e-05, 6.223241507541388e-05, 6.155147275421768e-05, 8.003221591934562e-05, 6.163583748275414e-05, 3.858302079606801e-05, 3.064595512114465e-05, 2.8436194043024443e-05, 2.627541107358411e-05, 2.4766464775893837e-05, 2.357139965170063e-05 ] } } }, "post_attention_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "90a633f12d984935028fe5d7212be48e7da7b4a0806751939a551938d16dc540", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.0933690071105957, 0.35791492462158203, 0.3940892517566681, 0.4057151973247528, 0.09380833059549332, 0.11286329478025436, 0.13231655955314636, 0.1621972620487213, 0.05156813934445381, 0.08533531427383423, 0.12003090232610703, 0.16063299775123596, 0.046871740370988846, 0.079384446144104, 0.11459959298372269, 0.15625107288360596, 0.08397205173969269, 0.11654024571180344, 0.15294702351093292, 0.21572710573673248, 0.03708760067820549, 0.06553087383508682, 0.09390194714069366, 0.12852425873279572, 0.025487789884209633, 0.06059965491294861, 0.08860157430171967, 0.12100546807050705, 0.024166326969861984, 0.06877554953098297, 0.12682415544986725, 0.2025238573551178 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 8.806617699974595e-06, "first_to_last_ratio": 1.56303737126734, "imbalance_abs_log_ratio": 0.44663096111341144, "last_quartile_mean": 5.634297593815063e-06, "mean": 6.4954337446465615e-06, "normalized": [ 1.5749560665320739, 1.7035419084923615, 1.473473597165368, 1.3203194560527711, 1.3082810426759777, 1.2337088574590254, 1.1650927296279558, 1.0671601331924767, 0.9989232512280567, 0.9181301596103544, 0.870035594807528, 0.8307099617546676, 0.8187320372884861, 0.7858094774713436, 0.7643921324856002, 0.7403421196819719, 0.7464426083440034, 0.7265594693619067, 0.7303426866670569, 0.7313091090305842, 1.1662257065266057, 1.144314508791491, 1.1279322355213752, 1.1138701040309826, 1.6760471782479454, 1.4686860119069318, 1.1046356737084946, 0.705013235389547, 0.5469797605030541, 0.5132313590313065, 0.4723390001843142, 0.4524628272283817 ], "peak_layer": 2, "peak_normalized": 1.7035419084923615, "population_cv": 0.3420755226148234, "rest_mean": 6.199918036347501e-06, "spike_contrast": 1.3050525058599312, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 8.09121856946149e-06, "top_five_layers": [ 2, 25, 1, 3, 26 ], "top_five_spike_overlap": 1 }, "values": [ 1.0230022780888248e-05, 1.106524359784089e-05, 9.570850124873687e-06, 8.576047548558563e-06, 8.497852832078934e-06, 8.013474143808708e-06, 7.5677826316677965e-06, 6.931667940079933e-06, 6.488439794338774e-06, 5.963653620710829e-06, 5.65125856155646e-06, 5.395821517595323e-06, 5.3180197028268594e-06, 5.104173396830447e-06, 4.9650584514893126e-06, 4.8088431867654435e-06, 4.848468506679637e-06, 4.719318894785829e-06, 4.743892532133032e-06, 4.750169864564668e-06, 7.575141808047192e-06, 7.432819074892905e-06, 7.326409104280174e-06, 7.23506946087582e-06, 1.0886653399211355e-05, 9.539752682030667e-06, 7.175087830546545e-06, 4.5793667595717125e-06, 3.552870794010232e-06, 3.333660288262763e-06, 3.068046680709813e-06, 2.9389423161774175e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 3.5046917446379666e-05, "first_to_last_ratio": 1.5558668967739346, "imbalance_abs_log_ratio": 0.44203288017821607, "last_quartile_mean": 2.2525652752847236e-05, "mean": 2.591035612908854e-05, "normalized": [ 1.5943755826994226, 1.724773497144586, 1.4816090565858486, 1.3124209364790316, 1.2902054208787557, 1.2106312392342944, 1.1486983445698478, 1.0582618354436555, 0.9932880900235477, 0.9159817519162186, 0.8686829207995, 0.8316453375345929, 0.8208867703533782, 0.7899750464002161, 0.7686771565699332, 0.7448321549511959, 0.7530948586655917, 0.728766437947155, 0.7302900575696902, 0.7267017624138756, 1.1650961226168133, 1.1455286709782952, 1.128276941445889, 1.1123503671452468, 1.6775513279007794, 1.4677338192142246, 1.1027363224128268, 0.7056584293154414, 0.5505982078262284, 0.5173269900840235, 0.47666941492738624, 0.4566751279525084 ], "peak_layer": 2, "peak_normalized": 1.724773497144586, "population_cv": 0.34224719066336146, "rest_mean": 2.473114374071075e-05, "spike_contrast": 1.3051601399734116, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.227810302632861e-05, "top_five_layers": [ 2, 25, 1, 3, 26 ], "top_five_spike_overlap": 1 }, "values": [ 4.13108391512651e-05, 4.46894955530297e-05, 3.838901830022223e-05, 3.40052938554436e-05, 3.3429681934649125e-05, 3.136788654956035e-05, 2.9763183192699216e-05, 2.7419941034168005e-05, 2.573644815129228e-05, 2.3733413399895653e-05, 2.250788384117186e-05, 2.154822686861735e-05, 2.126946856151335e-05, 2.0468534785322845e-05, 1.991669887502212e-05, 1.9298866391181946e-05, 1.9512955987011082e-05, 1.888259794213809e-05, 1.8922075469163246e-05, 1.8829101463779807e-05, 3.0188055461621843e-05, 2.968105582112912e-05, 2.9234057365101762e-05, 2.8821394153055735e-05, 4.346595233073458e-05, 3.8029505958547816e-05, 2.8572290830197744e-05, 1.828386120905634e-05, 1.4266195648815483e-05, 1.3404126548266504e-05, 1.2350674296612851e-05, 1.183261520054657e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.031079956330358982, "first_to_last_ratio": 1.5558668891156422, "imbalance_abs_log_ratio": 0.44203287525601326, "last_quartile_mean": 0.019975973875261843, "mean": 0.02297756270854734, "normalized": [ 1.5943756356110714, 1.724773491468354, 1.481608961955578, 1.3124209326195087, 1.290205357701199, 1.2106312502205263, 1.148698376834502, 1.0582618210377184, 0.9932880879345768, 0.9159817361210602, 0.8686829595918353, 0.8316453012075344, 0.8208867764933998, 0.7899750982973729, 0.7686771639206968, 0.7448322095954226, 0.7530948655254265, 0.7287664475791007, 0.7302900389200907, 0.7267017563604248, 1.1650961755697755, 1.1455287069742175, 1.128276902664199, 1.1123503269385118, 1.677551320090352, 1.4677337830096242, 1.1027363399282046, 0.7056584429149566, 0.5505981868647465, 0.5173269889474004, 0.4766694331791195, 0.45667512392349324 ], "peak_layer": 2, "peak_normalized": 1.724773491468354, "population_cv": 0.34224718411252225, "rest_mean": 0.021931825378150853, "spike_contrast": 1.305160140532808, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.02862454429268837, "top_five_layers": [ 2, 25, 1, 3, 26 ], "top_five_spike_overlap": 1 }, "values": [ 0.036634866148233414, 0.03963109105825424, 0.03404376283288002, 0.03015623427927494, 0.029645774513483047, 0.02781735546886921, 0.02639428898692131, 0.024316277354955673, 0.022823339328169823, 0.02104702778160572, 0.01996021717786789, 0.019109182059764862, 0.018861977383494377, 0.018151702359318733, 0.017662327736616135, 0.0171144288033247, 0.017304284498095512, 0.016745276749134064, 0.01678028516471386, 0.01669783517718315, 0.026771070435643196, 0.02632145769894123, 0.025925053283572197, 0.02555909939110279, 0.038546040654182434, 0.03372494503855705, 0.025338193401694298, 0.016214311122894287, 0.012651404365897179, 0.01188691332936287, 0.010952701792120934, 0.010493281297385693 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 1.35404550860585e-06, "first_to_last_ratio": 0.8953878937472707, "imbalance_abs_log_ratio": 0.11049825376335692, "last_quartile_mean": 1.512244601542534e-06, "mean": 1.3040982604906048e-06, "normalized": [ 2.110638840871199, 1.1249226908816714, 1.0256911491060132, 0.900748267309111, 0.8113140664581598, 0.783778905024042, 0.7743963503085244, 0.774911476771345, 0.751396289373451, 0.7213368426359936, 0.7107804987414867, 0.7118675908029816, 0.6961397137233883, 0.7013054932536184, 0.7136704026579705, 0.7165434814928291, 0.7075161713763967, 0.7313649399910757, 0.7600615551168772, 0.7962146454067395, 1.348771771425082, 1.4055438684819657, 1.4577635168986471, 1.48644627094684, 2.21381268234252, 1.987404445860277, 1.5423377972809362, 0.9473250623137235, 0.7099416420164079, 0.6750333462370576, 0.619627785796851, 0.5813924390968176 ], "peak_layer": 25, "peak_normalized": 2.21381268234252, "population_cv": 0.44502526707708084, "rest_mean": 1.1634325174041064e-06, "spike_contrast": 1.7737971410342603, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.0636932731576962e-06, "top_five_layers": [ 25, 1, 26, 27, 24 ], "top_five_spike_overlap": 2 }, "values": [ 2.752480440904037e-06, 1.467009724365198e-06, 1.3376020433497615e-06, 1.1746642485377379e-06, 1.0580332627796452e-06, 1.0221247066510841e-06, 1.0098889333676198e-06, 1.0105607088917168e-06, 9.798945939110126e-07, 9.406941217093845e-07, 9.269276119994174e-07, 9.28345286865806e-07, 9.078345897250983e-07, 9.145712738245493e-07, 9.30696330669889e-07, 9.344431077806803e-07, 9.226706083609315e-07, 9.537717460261774e-07, 9.911949518937035e-07, 1.0383421340520727e-06, 1.7589309209142812e-06, 1.8329673139305669e-06, 1.9010668665941921e-06, 1.9384719962545205e-06, 2.8870292680949206e-06, 2.5917706807376817e-06, 2.01136003852298e-06, 1.2354049658824806e-06, 9.258336604034412e-07, 8.80309812600899e-07, 8.080555176093185e-07, 7.581928684885497e-07 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 2.6826736302609788e-05, "first_to_last_ratio": 1.7808361848893863, "imbalance_abs_log_ratio": 0.5770830207899085, "last_quartile_mean": 1.5064123545016628e-05, "mean": 1.8887774274389812e-05, "normalized": [ 1.6688812801890411, 1.780234847804306, 1.5513349080163747, 1.3933355693978184, 1.3714991534523977, 1.2838711230533608, 1.204772837299277, 1.1086526212528207, 1.0355060883360716, 0.9557668919958734, 0.904713813394844, 0.8641098405178455, 0.8541488091515049, 0.8168843126808334, 0.791035295679702, 0.7641260074392275, 0.7785006543887889, 0.7464331675152748, 0.7506130024151308, 0.7465782030288268, 1.108381811250093, 1.0754895673127944, 1.0469354838957237, 1.0177188212401151, 1.571884494923229, 1.3089586073032224, 0.9748749838384205, 0.6421329388739745, 0.5159696177684924, 0.4848173184682084, 0.4500964328766437, 0.43174149523976274 ], "peak_layer": 2, "peak_normalized": 1.780234847804306, "population_cv": 0.34946420446589777, "rest_mean": 1.831385863490006e-05, "spike_contrast": 1.2005617803412982, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.1986918727634475e-05, "top_five_layers": [ 2, 1, 25, 3, 4 ], "top_five_spike_overlap": 1 }, "values": [ 3.152145291096531e-05, 3.3624673960730433e-05, 2.930126356659457e-05, 2.6317007723264396e-05, 2.59045664279256e-05, 2.4249467969639227e-05, 2.275547740282491e-05, 2.093998045893386e-05, 1.955840525624808e-05, 1.8052309314953163e-05, 1.708803029032424e-05, 1.6321111615980044e-05, 1.613296990399249e-05, 1.5429126506205648e-05, 1.4940896107873414e-05, 1.4432639545702841e-05, 1.4704144632560201e-05, 1.4098461178946309e-05, 1.4177408957039006e-05, 1.4101200576988049e-05, 2.093486546073109e-05, 2.031360418186523e-05, 1.9774281099671498e-05, 1.922244337038137e-05, 2.9689399525523186e-05, 2.4723314709262922e-05, 1.8413218640489504e-05, 1.212846200360218e-05, 9.745517672854476e-06, 9.15712007554248e-06, 8.501319825882092e-06, 8.154635906976182e-06 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 1.990490659409261e-05, "first_to_last_ratio": 1.8412792890601586, "imbalance_abs_log_ratio": 0.6104605958291677, "last_quartile_mean": 1.0810367939484422e-05, "mean": 1.377771818056317e-05, "normalized": [ 1.6665619998308319, 1.79086008317062, 1.5889877366066747, 1.4372834521538635, 1.430092633213083, 1.3165271765422786, 1.2085508236984301, 1.1188737887796325, 1.0465311084025242, 0.9650118361678015, 0.9220174298780283, 0.8760117542861069, 0.8686207869680962, 0.8297991310811854, 0.8079082323816149, 0.7776292551377597, 0.8073508930844662, 0.7726163040273158, 0.7836684287654929, 0.7851443250115728, 1.046365550312124, 0.999873696356647, 0.9598164939075269, 0.9168820925237239, 1.4947545536019826, 1.1927417423771507, 0.9013444525372633, 0.6362942673182214, 0.5473963720090704, 0.520763963533875, 0.4970577348957056, 0.48666190143933036 ], "peak_layer": 2, "peak_normalized": 1.79086008317062, "population_cv": 0.338908394256776, "rest_mean": 1.3564575662409576e-05, "spike_contrast": 1.1005643044156004, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.4928687778592576e-05, "top_five_layers": [ 2, 1, 3, 25, 4 ], "top_five_spike_overlap": 1 }, "values": [ 2.2961421564104967e-05, 2.4673965526744723e-05, 2.1892625227337703e-05, 1.980248634936288e-05, 1.970341327250935e-05, 1.8138740415452048e-05, 1.6651072655804455e-05, 1.541552774142474e-05, 1.4418810678762384e-05, 1.3295661119627766e-05, 1.2703296306426637e-05, 1.2069443073414732e-05, 1.1967612408625428e-05, 1.1432738574512769e-05, 1.113113194151083e-05, 1.0713956726249307e-05, 1.1123453077743761e-05, 1.0644889698596671e-05, 1.0797162758535706e-05, 1.0817497241077945e-05, 1.4416529666050337e-05, 1.3775978004559875e-05, 1.3224081158114132e-05, 1.2632543075596914e-05, 2.059430698864162e-05, 1.643325958866626e-05, 1.2418469850672409e-05, 8.76668309501838e-06, 7.541872946603689e-06, 7.174939128162805e-06, 6.848321390862111e-06, 6.705090527248103e-06 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 7.224441287689842e-05, "first_to_last_ratio": 1.5165559153010966, "imbalance_abs_log_ratio": 0.4164419187500513, "last_quartile_mean": 4.763715742228669e-05, "mean": 5.4423362598754466e-05, "normalized": [ 1.5775463275529453, 1.6582350809971804, 1.4282948011267544, 1.2844616698908673, 1.2647496153018294, 1.1989574712997115, 1.1495848200976218, 1.0577878819073543, 0.9907160399417638, 0.9262708408813498, 0.883942611455516, 0.8398077378143262, 0.8329893220354845, 0.7988235121104722, 0.7779925908001321, 0.7481533819347611, 0.7515440735091025, 0.7261628179024023, 0.7235409879022285, 0.7209281152530318, 1.190361088197821, 1.1639970641279485, 1.1557592424487526, 1.1469358506609055, 1.6644766166347367, 1.4764898949048717, 1.1319516730861015, 0.7079687000744663, 0.5620484392150152, 0.5221173372870457, 0.4823503420504847, 0.45505405159701545 ], "peak_layer": 25, "peak_normalized": 1.6644766166347367, "population_cv": 0.3330112829863002, "rest_mean": 5.175958115917941e-05, "spike_contrast": 1.3293728587341342, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.880778237245977e-05, "top_five_layers": [ 25, 2, 1, 26, 3 ], "top_five_spike_overlap": 1 }, "values": [ 8.585537580074742e-05, 9.024672908708453e-05, 7.773260585963726e-05, 6.990472320467234e-05, 6.883192691020668e-05, 6.525129720102996e-05, 6.256427150219679e-05, 5.756837344961241e-05, 5.3918098274152726e-05, 5.0410773837938905e-05, 4.810712925973348e-05, 4.5705161028308794e-05, 4.5334079914027825e-05, 4.347466165199876e-05, 4.2340972868259996e-05, 4.071702278451994e-05, 4.090155562153086e-05, 3.952022234443575e-05, 3.9377533539664e-05, 3.92353322240524e-05, 6.478345312643796e-05, 6.334863428492099e-05, 6.290030432865024e-05, 6.242010567802936e-05, 9.05864144442603e-05, 8.03555449238047e-05, 6.160461634863168e-05, 3.853003727272153e-05, 3.058856600546278e-05, 2.841538116626907e-05, 2.6251127565046772e-05, 2.4765571652096696e-05 ] } } }, "post_mlp_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "89f2226456bfe96e245d2aad0f699d32c676f9f53099789cd870253d002402f5", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.16915453970432281, 0.374446302652359, 0.39966297149658203, 0.4194087088108063, 0.09892481565475464, 0.11664740741252899, 0.13951337337493896, 0.18593810498714447, 0.06607726216316223, 0.10425819456577301, 0.14551182091236115, 0.19865906238555908, 0.06484340131282806, 0.10430748760700226, 0.14720691740512848, 0.20439592003822327, 0.10052042454481125, 0.14085541665554047, 0.1745898574590683, 0.2420409768819809, 0.049456026405096054, 0.08509506285190582, 0.11783231794834137, 0.15649692714214325, 0.05533855780959129, 0.08507220447063446, 0.11942370235919952, 0.1559232622385025, 0.06628894805908203, 0.12384890764951706, 0.19928614795207977, 0.30097496509552 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 8.590702691435581e-06, "first_to_last_ratio": 1.8480715518500432, "imbalance_abs_log_ratio": 0.6141426910035225, "last_quartile_mean": 4.64846866066182e-06, "mean": 6.141734282039124e-06, "normalized": [ 1.5093899710851086, 1.7774627674104075, 1.5466131926073605, 1.3587806393466884, 1.376064007570294, 1.2923146097688787, 1.2090959462619588, 1.12021616630854, 1.004924506733677, 0.936409656192841, 0.8893813258978764, 0.8522756833460008, 0.8489903580975298, 0.8144462785551897, 0.7899504622809753, 0.7690807873765435, 0.7833439013302343, 0.7615577324111179, 0.7677689822013803, 0.7793140079922765, 1.2223161762120276, 1.1953412799087557, 1.1812886678522816, 1.158746454306897, 1.5665908074141255, 1.174785173606052, 0.7515966908225283, 0.5787833701839877, 0.542591296980972, 0.5001098224743231, 0.47913722894972555, 0.46133204851344506 ], "peak_layer": 2, "peak_normalized": 1.7774627674104075, "population_cv": 0.34153331518044594, "rest_mean": 5.8404973683705236e-06, "spike_contrast": 1.330094532345783, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 7.768413615849567e-06, "top_five_layers": [ 2, 25, 3, 1, 5 ], "top_five_spike_overlap": 1 }, "values": [ 9.270272130379453e-06, 1.0916704013652634e-05, 9.498887266090605e-06, 8.345269634446595e-06, 8.451419489574619e-06, 7.937052941997536e-06, 7.425946023431607e-06, 6.8800700319116e-06, 6.17197929386748e-06, 5.751179287472041e-06, 5.462343779072398e-06, 5.234450782154454e-06, 5.214273187448271e-06, 5.002112629881594e-06, 4.85166583530372e-06, 4.7234898374881595e-06, 4.8110900934261736e-06, 4.67728523290134e-06, 4.715433078672504e-06, 4.786339559359476e-06, 7.507141162932385e-06, 7.34146851755213e-06, 7.255161108332686e-06, 7.116712822607951e-06, 9.621584467822686e-06, 7.2152183747675736e-06, 4.6161071622918826e-06, 3.5547336665331386e-06, 3.332451569804107e-06, 3.0715416414750507e-06, 2.942733544841758e-06, 2.833378857758362e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 3.415737637624261e-05, "first_to_last_ratio": 1.8366154583232197, "imbalance_abs_log_ratio": 0.6079244529772941, "last_quartile_mean": 1.859800113379606e-05, "mean": 2.449507587698463e-05, "normalized": [ 1.5280789932038084, 1.7998272225327347, 1.5548658405253786, 1.3487269708054683, 1.355959686916098, 1.2670462453974145, 1.1917295497505567, 1.1094366299276686, 1.0002759200476248, 0.9350908043362248, 0.8890664678445747, 0.8543900058444925, 0.8521964577101172, 0.8202362556937576, 0.7954410452239409, 0.774118055332584, 0.790080185568276, 0.7637872374525602, 0.7672650274853033, 0.7737128218389219, 1.2208592808406884, 1.1957515868558763, 1.1801926851103044, 1.1578275389334327, 1.56661085423705, 1.1729442260089005, 0.7501394011201082, 0.5827289570399494, 0.5468132172024143, 0.5045648964555371, 0.48383431085146067, 0.46640162190677265 ], "peak_layer": 2, "peak_normalized": 1.7998272225327347, "population_cv": 0.3408989522980816, "rest_mean": 2.3296412109630182e-05, "spike_contrast": 1.3292974074706234, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.096786022069864e-05, "top_five_layers": [ 2, 25, 3, 1, 5 ], "top_five_spike_overlap": 1 }, "values": [ 3.7430410884553567e-05, 4.408690438140184e-05, 3.808655674220063e-05, 3.303716948721558e-05, 3.321433541714214e-05, 3.1036393920658156e-05, 2.9191505745984614e-05, 2.717573443078436e-05, 2.450183455948718e-05, 2.2905120204086415e-05, 2.177775058953557e-05, 2.0928348021698184e-05, 2.0874616893706843e-05, 2.0091749320272356e-05, 1.9484388758428395e-05, 1.896208050311543e-05, 1.9353074094397016e-05, 1.8709026335272938e-05, 1.87942150660092e-05, 1.895215427794028e-05, 2.9905040719313547e-05, 2.9290025850059465e-05, 2.8908909371239133e-05, 2.8361073418636806e-05, 3.837425174424425e-05, 2.8731357815559022e-05, 1.837472154875286e-05, 1.4273990018409677e-05, 1.3394231245911215e-05, 1.2359355423541274e-05, 1.1851558156195097e-05, 1.1424543117755093e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.030291102826595306, "first_to_last_ratio": 1.836615443594157, "imbalance_abs_log_ratio": 0.6079244449576171, "last_quartile_mean": 0.01649289345368743, "mean": 0.021722478268202394, "normalized": [ 1.5280789950763196, 1.7998271252848101, 1.5548657841468059, 1.348726988674529, 1.3559596926852158, 1.2670462836243699, 1.1917295760876983, 1.1094366348920448, 1.0002759591187138, 0.9350908165690212, 0.8890665216893189, 0.854389946593731, 0.8521964433951469, 0.8202362588952334, 0.7954410379921495, 0.7741180706570835, 0.7900801988352641, 0.7637872288070612, 0.7672649702060752, 0.7737128286369706, 1.220859326004051, 1.1957515559762715, 1.1801926976289394, 1.1578275568887841, 1.5666109429413357, 1.1729442161041392, 0.7501394218567367, 0.5827289357023024, 0.5468131826436037, 0.5045649013522854, 0.48383431964903284, 0.4664015813849557 ], "peak_layer": 2, "peak_normalized": 1.7998271252848101, "population_cv": 0.3408989541403313, "rest_mean": 0.020659491143844747, "spike_contrast": 1.3292974424452781, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.027462608739733697, "top_five_layers": [ 2, 25, 3, 1, 5 ], "top_five_spike_overlap": 1 }, "values": [ 0.03319366276264191, 0.03909670561552048, 0.033775538206100464, 0.029297692701220512, 0.029454804956912994, 0.027523385360836983, 0.025887319818139076, 0.024099713191390038, 0.021728472784161568, 0.020312489941716194, 0.019312728196382523, 0.018559467047452927, 0.01851181872189045, 0.01781756430864334, 0.017278950661420822, 0.01681576296687126, 0.017162499949336052, 0.016591351479291916, 0.016666896641254425, 0.016806960105895996, 0.02652009017765522, 0.025974687188863754, 0.025636710226535797, 0.025150883942842484, 0.034030672162771225, 0.025479255244135857, 0.01629488728940487, 0.012658316642045975, 0.011878137476742268, 0.010960400104522705, 0.010510080493986607, 0.01013139821588993 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 1.2902427783956227e-06, "first_to_last_ratio": 1.029048197590643, "imbalance_abs_log_ratio": 0.02863429500731451, "last_quartile_mean": 1.2538215230506466e-06, "mean": 1.2122287866844772e-06, "normalized": [ 2.055483116490403, 1.16592724455194, 1.0705361054058493, 0.9040107885163775, 0.8620930215951813, 0.8310378012580824, 0.818375083886064, 0.8073834914564767, 0.775268285168324, 0.7515936243867247, 0.7423653459011068, 0.7364406316816996, 0.7333301672671342, 0.735517569374843, 0.7447337029207872, 0.7514944016363407, 0.7526459669973662, 0.7784140121104363, 0.8099856971794487, 0.85455659416526, 1.4396979119755318, 1.4928617992512132, 1.5516612198854238, 1.560098717434144, 2.147199074972933, 1.6658229963345421, 1.0896437999535653, 0.7693256114435307, 0.7162721540247274, 0.6628331064942004, 0.6315260779484237, 0.5918648783319196 ], "peak_layer": 25, "peak_normalized": 2.147199074972933, "population_cv": 0.4080611230121109, "rest_mean": 1.0689380137254947e-06, "spike_contrast": 1.8579177980033843, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.9859989606629826e-06, "top_five_layers": [ 25, 1, 26, 24, 23 ], "top_five_spike_overlap": 3 }, "values": [ 2.491715804353589e-06, 1.4133705690255738e-06, 1.2977346841580584e-06, 1.0958679013128858e-06, 1.0450539775774814e-06, 1.007407945508021e-06, 9.920578349920106e-07, 9.787335102373618e-07, 9.398025326845527e-07, 9.11103427370108e-07, 8.99916642538301e-07, 8.927345334086567e-07, 8.889639389053627e-07, 8.916155707083817e-07, 9.027876330947038e-07, 9.109831466957985e-07, 9.123791073761822e-07, 9.4361587343883e-07, 9.818879789236235e-07, 1.0359181032981724e-06, 1.745243253026274e-06, 1.8096900475939037e-06, 1.880968397927063e-06, 1.8911965753432014e-06, 2.60289652942447e-06, 2.019358589677722e-06, 1.3208975815359736e-06, 9.325986525254848e-07, 8.682857242092723e-07, 8.035053724597674e-07, 7.655540912310244e-07, 7.174756433414586e-07 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 2.623750151542481e-05, "first_to_last_ratio": 2.0853538348087515, "imbalance_abs_log_ratio": 0.734938545796755, "last_quartile_mean": 1.2581798387145682e-05, "mean": 1.7980496579639293e-05, "normalized": [ 1.6149711389621628, 1.8440799819696896, 1.618964509634521, 1.4287722646527024, 1.4327737284915447, 1.336085275494912, 1.244362688288269, 1.1537509893461335, 1.0396425013156374, 0.9720423049665666, 0.9222927001083071, 0.8842527943419762, 0.8820887827887408, 0.8438473571769589, 0.8142897980379072, 0.7901851109820188, 0.8119852688815937, 0.7786975693100754, 0.7846054796788003, 0.7927060817594883, 1.155305180804801, 1.1157534715392625, 1.087167800530356, 1.053401288996729, 1.4315095758867469, 1.0505721203230916, 0.6869651691390706, 0.5438924522373797, 0.5099821858340812, 0.47546948435063124, 0.45736400869786514, 0.44222093547197916 ], "peak_layer": 2, "peak_normalized": 1.8440799819696896, "population_cv": 0.35430701504424944, "rest_mean": 1.741901407378329e-05, "spike_contrast": 1.206296867449395, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.101250211126171e-05, "top_five_layers": [ 2, 3, 1, 5, 25 ], "top_five_spike_overlap": 1 }, "values": [ 2.9037983040325344e-05, 3.3157473808387294e-05, 2.9109785828040913e-05, 2.5690034817671403e-05, 2.5761983124539256e-05, 2.402347672614269e-05, 2.237425906059798e-05, 2.0745015717693605e-05, 1.869328843895346e-05, 1.7477803339716047e-05, 1.6583280739723705e-05, 1.5899304344202392e-05, 1.5860394341871142e-05, 1.5172794519457966e-05, 1.4641334928455763e-05, 1.4207920685294084e-05, 1.4599898349842988e-05, 1.4001368981553242e-05, 1.4107596143730916e-05, 1.4253248991735745e-05, 2.077296085190028e-05, 2.0061801478732377e-05, 1.954781691893004e-05, 1.894067827379331e-05, 2.5739253032952547e-05, 1.8889808416133747e-05, 1.2351974874036387e-05, 9.779456377145834e-06, 9.169732948066667e-06, 8.549177437089384e-06, 8.22363199404208e-06, 7.95135201769881e-06 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 1.9581527340051252e-05, "first_to_last_ratio": 2.114816072993664, "imbalance_abs_log_ratio": 0.7489678455931664, "last_quartile_mean": 9.259210571599397e-06, "mean": 1.3228676394305694e-05, "normalized": [ 1.6500735780731406, 1.8400111605581244, 1.6493108461171446, 1.4735313122799243, 1.4771768052702559, 1.3591396693434894, 1.2350307910368918, 1.157592253211377, 1.0479722569915417, 0.982576963868735, 0.9366789805891723, 0.8926124752775004, 0.8930667868662171, 0.8537537778081574, 0.8259643189462823, 0.8007439756181789, 0.8348537832014102, 0.7976905039793059, 0.8145445151464921, 0.8327822928612116, 1.0854895389936556, 1.0299067711716956, 0.9881085924711813, 0.941909889020758, 1.3133342259244611, 0.9691397086071862, 0.6727659548663321, 0.5716326407523367, 0.5432097376073742, 0.5178674286681353, 0.5091556876964263, 0.5023727771759046 ], "peak_layer": 2, "peak_normalized": 1.8400111605581244, "population_cv": 0.3443028950134506, "rest_mean": 1.3052906962440358e-05, "spike_contrast": 1.08618190317108, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.417783132637851e-05, "top_five_layers": [ 2, 1, 3, 5, 4 ], "top_five_spike_overlap": 0 }, "values": [ 2.182828939112369e-05, 2.4340912204934284e-05, 2.181819945690222e-05, 1.9492868887027726e-05, 1.9541093934094533e-05, 1.7979618860408664e-05, 1.633782267163042e-05, 1.5313413314288482e-05, 1.3863285857951269e-05, 1.2998192687518895e-05, 1.2391023119562306e-05, 1.1808081580966245e-05, 1.1814091521955561e-05, 1.1294032447040081e-05, 1.0926414688583463e-05, 1.0592782928142697e-05, 1.10440105345333e-05, 1.0552389539952856e-05, 1.0775345799629577e-05, 1.1016607459168881e-05, 1.4359589840751141e-05, 1.3624303392134607e-05, 1.3071368812234141e-05, 1.2460221114451997e-05, 1.737367347232066e-05, 1.2820435586036183e-05, 8.899803106032778e-06, 7.5619432209350634e-06, 7.185945833043661e-06, 6.850700629001949e-06, 6.735455826856196e-06, 6.645726898568682e-06 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 7.043373079795856e-05, "first_to_last_ratio": 1.794913571378016, "imbalance_abs_log_ratio": 0.5849568711308802, "last_quartile_mean": 3.924073666894401e-05, "mean": 5.140230501865517e-05, "normalized": [ 1.5087403105240826, 1.7376064767309243, 1.5068187794842454, 1.3277036351725697, 1.3267897932283201, 1.2566328902109432, 1.193565905920887, 1.10409899605831, 0.9958007281322062, 0.9449997646909633, 0.9003146488278224, 0.8682329364457932, 0.8598045280306121, 0.830875683595643, 0.8040099902230055, 0.7758182634963202, 0.787675702441156, 0.7578673417545456, 0.7607281936032495, 0.7691491706829174, 1.248063781843379, 1.219064021235227, 1.210815380364841, 1.1975893017423487, 1.5568135491842439, 1.198936284385081, 0.7506373600464573, 0.5962251073745872, 0.5530341853719639, 0.5112488474006913, 0.4817714642429133, 0.4585669775537495 ], "peak_layer": 2, "peak_normalized": 1.7376064767309243, "population_cv": 0.33250107451051913, "rest_mean": 4.8675420287138504e-05, "spike_contrast": 1.3585395293714189, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.612748256884515e-05, "top_five_layers": [ 2, 25, 1, 3, 4 ], "top_five_spike_overlap": 1 }, "values": [ 7.755272963549942e-05, 8.931697811931372e-05, 7.745395851088688e-05, 6.82470272295177e-05, 6.820005364716053e-05, 6.459382711909711e-05, 6.135203875601292e-05, 5.675323336618021e-05, 5.118645276525058e-05, 4.857516614720225e-05, 4.6278248191811144e-05, 4.4629174226429313e-05, 4.4195934606250376e-05, 4.2708925320766866e-05, 4.132796675548889e-05, 3.987884701928124e-05, 4.048834671266377e-05, 3.895612826454453e-05, 3.910318264388479e-05, 3.9536040276288986e-05, 6.415335519704968e-05, 6.266270065680146e-05, 6.223870150279254e-05, 6.155885057523847e-05, 8.002380491234362e-05, 6.162808858789504e-05, 3.858449053950608e-05, 3.064734482904896e-05, 2.8427231882233173e-05, 2.6279369194526225e-05, 2.4764163754298352e-05, 2.357139965170063e-05 ] } } }, "pre_attention_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "ee695e78937de01b7128d3866b8fc5dd328ebdf2c159be37918d4f02952d5248", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.0626930519938469, 0.09030644595623016, 0.20229491591453552, 0.20300470292568207, 0.19049881398677826, 0.12695962190628052, 0.12132706493139267, 0.11585728824138641, 0.13427142798900604, 0.10485932976007462, 0.10419609397649765, 0.10852954536676407, 0.1286265105009079, 0.09096142649650574, 0.10080574452877045, 0.10566592961549759, 0.1287592649459839, 0.09650235623121262, 0.1065765842795372, 0.11446042358875275, 0.12395420670509338, 0.07221997529268265, 0.07677769660949707, 0.08416415005922318, 0.09094838052988052, 0.059439923614263535, 0.062182098627090454, 0.06896103918552399, 0.0776243805885315, 0.06285123527050018, 0.07231374084949493, 0.08016443997621536 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 6.445229900009508e-06, "first_to_last_ratio": 3.2624438520048176, "imbalance_abs_log_ratio": 1.1824765623872502, "last_quartile_mean": 1.975583394653313e-06, "mean": 4.253825524713761e-06, "normalized": [ 2.4840437282309926, 1.9797666001969463, 1.4671055925735577, 1.2878190669487082, 0.5260608364925744, 1.1768604383807315, 1.614586292818476, 1.5850442485566052, 1.0988243455043247, 1.005618695551378, 0.7942281382078725, 0.6973029000313798, 0.4799813024385088, 0.9271140461656938, 0.7528430956596607, 0.709128152394714, 0.6908101371769495, 1.1851087647750558, 1.095072579432435, 1.341180950527869, 0.8969825379311661, 1.7066434822240424, 1.391920286015404, 1.3905528878155025, 0.8417461610570187, 0.8612295275199655, 0.5210392747973219, 0.44218959549293074, 0.2528482673083683, 0.19503209023995946, 0.2745771611764032, 0.32673881635748375 ], "peak_layer": 1, "peak_normalized": 2.4840437282309926, "population_cv": 0.5257857184710876, "rest_mean": 4.060379602058809e-06, "spike_contrast": 1.304910876895338, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.298433507050504e-06, "top_five_layers": [ 1, 2, 22, 7, 8 ], "top_five_spike_overlap": 1 }, "values": [ 1.056668861565413e-05, 8.421581696893554e-06, 6.240811217139708e-06, 5.478157618199475e-06, 2.2377710138243856e-06, 5.006158971809782e-06, 6.8681683842442e-06, 6.742501682310831e-06, 4.67420704808319e-06, 4.277726475265808e-06, 3.378507926754537e-06, 2.966204874610412e-06, 2.041756715698284e-06, 3.943781393900281e-06, 3.2024631764215883e-06, 3.0165074349497445e-06, 2.9385857942543225e-06, 5.041245913162129e-06, 4.658247689803829e-06, 5.7051497606153134e-06, 3.815607215074124e-06, 7.259763606271008e-06, 5.920986041019205e-06, 5.915169367654016e-06, 3.580641305234167e-06, 3.663520146801602e-06, 2.2164101665111957e-06, 1.880997388070682e-06, 1.0755724133559852e-06, 8.296324836010172e-07, 1.1680033367156284e-06, 1.3898899169362267e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 2.7453660095488885e-05, "first_to_last_ratio": 3.5530147570645223, "imbalance_abs_log_ratio": 1.2677964704399247, "last_quartile_mean": 7.726863515244986e-06, "mean": 1.7375418202902893e-05, "normalized": [ 3.0990329757572086, 2.0234279828190296, 1.4545791466765596, 1.3058531494199213, 0.5113068185098264, 1.1682225634318304, 1.5404182003883868, 1.5373890670589336, 1.0336032092076375, 0.9771503605108718, 0.7985619347203995, 0.6840547276785537, 0.4691692550686129, 0.9148465264405745, 0.735399610659842, 0.6941989481597705, 0.6945366700924062, 1.2601673072481572, 1.0795398826722087, 1.2919510657973863, 0.8695908452797219, 1.6048235316923265, 1.3415380883224273, 1.3530312074713475, 0.8079477464784359, 0.8382032766418568, 0.506298777030152, 0.38799552207484617, 0.24613548184932987, 0.1923382806760905, 0.2622676451493263, 0.3164201950160222 ], "peak_layer": 1, "peak_normalized": 3.0990329757572086, "population_cv": 0.5859358902941397, "rest_mean": 1.6746729611609395e-05, "spike_contrast": 1.2402622528454206, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.0770336595887785e-05, "top_five_layers": [ 1, 2, 22, 7, 8 ], "top_five_spike_overlap": 1 }, "values": [ 5.384699397836812e-05, 3.515790740493685e-05, 2.527392098272685e-05, 2.2689744582748972e-05, 8.884169801604003e-06, 2.0298355593695305e-05, 2.6765410439111292e-05, 2.6712777980719693e-05, 1.7959288015845232e-05, 1.6978396160993725e-05, 1.3875347576686181e-05, 1.1885736967087723e-05, 8.152012014761567e-06, 1.589584098837804e-05, 1.2777875781466719e-05, 1.2061997040291317e-05, 1.2067865100107156e-05, 2.1895933969062753e-05, 1.8757456928142346e-05, 2.2448190065915696e-05, 1.5109504602150992e-05, 2.7884480005013756e-05, 2.330978531972505e-05, 2.3509483071393333e-05, 1.4038429981155787e-05, 1.4564132470695768e-05, 8.797152986517176e-06, 6.741584456904093e-06, 4.276706931705121e-06, 3.3419580631743884e-06, 4.557010015560081e-06, 5.497933216247475e-06 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.024346180609427392, "first_to_last_ratio": 3.5530147272876627, "imbalance_abs_log_ratio": 1.2677964620591942, "last_quartile_mean": 0.006852259975858033, "mean": 0.015408694773213938, "normalized": [ 3.0990329322739507, 2.0234280371042224, 1.4545791561021846, 1.3058531395175383, 0.5113068260075372, 1.1682225864246902, 1.54041822717383, 1.5373890267463077, 1.033603222232021, 0.977150441552234, 0.7985619170457312, 0.6840547199040246, 0.46916922043561676, 0.9148465134645866, 0.735399600665022, 0.6941989571631653, 0.6945366430801725, 1.260167323500315, 1.079539869921981, 1.2919510207112335, 0.869590858655765, 1.6048235807067084, 1.3415380851979657, 1.3530311320013937, 0.8079477360327032, 0.8382032941636696, 0.5062988054475902, 0.3879955278148233, 0.24613548429904128, 0.19233828573713255, 0.2622676326690507, 0.316420196247792 ], "peak_layer": 1, "peak_normalized": 3.0990329322739507, "population_cv": 0.5859358884323225, "rest_mean": 0.01485116745310801, "spike_contrast": 1.2402622460452561, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.018419342301785945, "top_five_layers": [ 1, 2, 22, 7, 8 ], "top_five_spike_overlap": 1 }, "values": [ 0.047752052545547485, 0.031178385019302368, 0.022413166239857674, 0.020121492445468903, 0.007878570817410946, 0.018000785261392593, 0.023735834285616875, 0.023689158260822296, 0.015926476567983627, 0.015056612901389599, 0.01230479683727026, 0.010540390387177467, 0.007229285314679146, 0.01409659069031477, 0.011331547982990742, 0.01069669984281063, 0.010701903142035007, 0.0194175336509943, 0.016634300351142883, 0.019907278940081596, 0.013399260118603706, 0.024728236719965935, 0.02067135088145733, 0.02084844373166561, 0.012449420057237148, 0.01291561871767044, 0.007801403757184744, 0.005978504661470652, 0.0037926265504211187, 0.0029636819381266832, 0.004041201900690794, 0.004875622224062681 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 2.676619267560909e-06, "first_to_last_ratio": 1.9273727589355951, "imbalance_abs_log_ratio": 0.65615781075397, "last_quartile_mean": 1.388739804042416e-06, "mean": 1.5331166416387987e-06, "normalized": [ 8.960860196102615, 2.4165862019701803, 0.40015430442207295, 0.5020167462088402, 0.27531974765435097, 0.42290788736073803, 0.43422624542344546, 0.5548725803559044, 0.46989516846172, 0.4961353645087654, 0.47806197857827515, 0.44403375140076745, 0.3170275927129732, 0.5465753703084478, 0.5167735913889384, 0.5340779254664982, 0.38275649914013377, 0.7500716739632997, 0.7130048725642506, 0.8553899549196788, 0.5458724638853519, 1.14906550427753, 1.2271579703744806, 1.3605333333073795, 1.1525698031314433, 1.5384187123474458, 1.2245440394309073, 1.0591028996880525, 0.5582665379546677, 0.5899862377231015, 0.5591831193285128, 0.5645517256392313 ], "peak_layer": 1, "peak_normalized": 8.960860196102615, "population_cv": 1.4966245656934398, "rest_mean": 1.5084051251842876e-06, "spike_contrast": 1.104848294843568, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.6665588304931589e-06, "top_five_layers": [ 1, 2, 26, 24, 23 ], "top_five_spike_overlap": 2 }, "values": [ 1.3738043890043627e-05, 3.7049085221951827e-06, 6.13483223332878e-07, 7.696502279941342e-07, 4.2209728690068005e-07, 6.483671199930541e-07, 6.657194830950175e-07, 8.506843869326985e-07, 7.204041025943297e-07, 7.606333838339197e-07, 7.329247750931245e-07, 6.807555337218218e-07, 4.860402782469464e-07, 8.379637961297703e-07, 7.922741929178301e-07, 8.188037554646144e-07, 5.868103585271456e-07, 1.1499473657750059e-06, 1.0931196356978035e-06, 1.3114125749780214e-06, 8.368861585950071e-07, 1.7616514469409594e-06, 1.8813763063008082e-06, 2.08585629479785e-06, 1.7670239458311698e-06, 2.358575329708401e-06, 1.8773688452711212e-06, 1.6237282807196607e-06, 8.558877198083792e-07, 9.045177193911513e-07, 8.572929459660372e-07, 8.655236456434068e-07 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 1.6779027987467998e-05, "first_to_last_ratio": 3.243468580915959, "imbalance_abs_log_ratio": 1.1766433068400135, "last_quartile_mean": 5.17317420190011e-06, "mean": 1.1421494306773639e-05, "normalized": [ 1.8750000502353934, 2.0742933204317313, 1.499766787707608, 1.3230502426683042, 0.6014560778841874, 1.28865529626601, 1.638374916391975, 1.4520006739045357, 1.1003723203487805, 1.0288735708977754, 0.846079816100661, 0.7379708109433667, 0.5286131723925126, 0.9886425346719802, 0.8001894688671397, 0.7148567418904531, 0.7248350301099004, 1.2847567659674841, 1.1450317455398444, 1.315651093666941, 0.8632584965988698, 1.6559906864936933, 1.4877647801980194, 1.4010498339942032, 0.8373095164556784, 0.9316683149495413, 0.5268388545667894, 0.34911029062222065, 0.263178844852241, 0.21027079669045023, 0.2273088703445351, 0.2777802773471742 ], "peak_layer": 2, "peak_normalized": 2.0742933204317313, "population_cv": 0.48938183499839527, "rest_mean": 1.089467859519236e-05, "spike_contrast": 1.3094740725630976, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.4266299149312545e-05, "top_five_layers": [ 2, 1, 22, 7, 3 ], "top_five_spike_overlap": 1 }, "values": [ 2.141530239896383e-05, 2.3691529349889606e-05, 1.7129577827290632e-05, 1.5111210814211518e-05, 6.869527169328649e-06, 1.471836912969593e-05, 1.871268977993168e-05, 1.658401743043214e-05, 1.2567896192194894e-05, 1.1751273632398807e-05, 9.663495802669786e-06, 8.428729415754788e-06, 6.037552338966634e-06, 1.129177508119028e-05, 9.139359463006258e-06, 8.164732207660563e-06, 8.278699169750325e-06, 1.467384208808653e-05, 1.3077973562758416e-05, 1.5026701476017479e-05, 9.859702004177961e-06, 1.8913888197857887e-05, 1.6992496966850013e-05, 1.6002082702470943e-05, 9.563325875205919e-06, 1.0641044354997575e-05, 6.01728697802173e-06, 3.987361196777783e-06, 3.0058956781431334e-06, 2.4016067072807346e-06, 2.5962069685192546e-06, 3.1726658562547527e-06 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 1.0303081751317222e-05, "first_to_last_ratio": 2.9721744199371685, "imbalance_abs_log_ratio": 1.0892938128646694, "last_quartile_mean": 3.466513163630225e-06, "mean": 7.48914104775622e-06, "normalized": [ 1.134846576669906, 2.104750427057885, 1.4893026181528508, 1.2855927575857782, 0.6955576756994144, 1.3655269588294994, 1.668331863538201, 1.26197998007218, 1.127631112658579, 1.1314887111491925, 0.9034127696390467, 0.7829496780483719, 0.590241168605664, 1.0620723405390324, 0.8439094495433462, 0.7337645382818543, 0.7490818699369026, 1.3150568434725778, 1.2184296179655392, 1.292534370358436, 0.8131031889825889, 1.6931476434332888, 1.639525627833163, 1.3947867796238638, 0.8329654796652215, 1.0187737272651016, 0.5507787193462383, 0.31951905984365025, 0.2822632079264078, 0.23119137885560714, 0.21655281573081742, 0.2509310436897934 ], "peak_layer": 2, "peak_normalized": 2.104750427057885, "population_cv": 0.4679813416395611, "rest_mean": 7.108157702801961e-06, "spike_contrast": 1.3430274776748572, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.546451110509224e-06, "top_five_layers": [ 2, 22, 7, 23, 3 ], "top_five_spike_overlap": 2 }, "values": [ 8.49902608024422e-06, 1.5762772818561643e-05, 1.1153597370139323e-05, 9.627985491533764e-06, 5.209129540162394e-06, 1.0226623999187723e-05, 1.249437264050357e-05, 9.451146070205141e-06, 8.444988452538382e-06, 8.4738785517402e-06, 6.7657856561709195e-06, 5.863620572199579e-06, 4.4203993638802785e-06, 7.95400956121739e-06, 6.320156899164431e-06, 5.495266123034526e-06, 5.609979780274443e-06, 9.84864618658321e-06, 9.12499126570765e-06, 9.679972208687104e-06, 6.08944446867099e-06, 1.2680221516347956e-05, 1.2278638678253628e-05, 1.0445754924148787e-05, 6.23819596512476e-06, 7.629740139236674e-06, 4.124859515286516e-06, 2.3929233066155575e-06, 2.1139089767530095e-06, 1.731424845274887e-06, 1.6217945812968537e-06, 1.8792579794535413e-06 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 5.228629038356303e-05, "first_to_last_ratio": 3.5341221177984883, "imbalance_abs_log_ratio": 1.2624649284217375, "last_quartile_mean": 1.4794703929510433e-05, "mean": 3.38591724471371e-05, "normalized": [ 2.3175815110773987, 2.0849991753893353, 1.5444936241572143, 1.3884269655042218, 0.5325768521618691, 1.2450562606715279, 1.5943115141923967, 1.6463800422841095, 1.0485863135218718, 0.9317844623970511, 0.8076061227472372, 0.713303077356814, 0.47980107124124943, 0.952419483673168, 0.7816031787468993, 0.6851375997161561, 0.7228547242403038, 1.2800238290212524, 1.070252478800878, 1.3050541862221579, 0.9347534737789246, 1.6461054143721179, 1.3910358232237334, 1.4002673394875913, 0.8348660589003104, 0.8664937714870038, 0.49220461509969915, 0.35739618216199814, 0.24500949045561318, 0.1901699978907823, 0.22605283144719793, 0.2833925285719155 ], "peak_layer": 1, "peak_normalized": 2.3175815110773987, "population_cv": 0.5302819872289567, "rest_mean": 3.2345506783544637e-05, "spike_contrast": 1.2994994115201224, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.2032967030536383e-05, "top_five_layers": [ 1, 2, 8, 22, 7 ], "top_five_spike_overlap": 1 }, "values": [ 7.847139204386622e-05, 7.059634663164616e-05, 5.229527596384287e-05, 4.7010988055262715e-05, 1.8032611478702165e-05, 4.215657463646494e-05, 5.398206849349663e-05, 5.5745065765222535e-05, 3.5504264815244824e-05, 3.1549450795864686e-05, 2.7344874979462475e-05, 2.415185190329794e-05, 1.6245667211478576e-05, 3.224813553970307e-05, 2.6464436814421788e-05, 2.3198192138806917e-05, 2.447526276228018e-05, 4.334054756327532e-05, 3.623786324169487e-05, 4.4188054744154215e-05, 3.164997906424105e-05, 5.573576709139161e-05, 4.709932181867771e-05, 4.741189331980422e-05, 2.8267873858567327e-05, 2.933876203314867e-05, 1.6665640941937454e-05, 1.2101138963771518e-05, 8.295818588521797e-06, 6.438998752855696e-06, 7.65396180213429e-06, 9.595436495146714e-06 ] } } }, "pre_mlp_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "e53fa425d89dfeb0e9395eca9bcd73dc0a55314abf45c5b10b43e2cbfa724345", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.059514161199331284, 0.08157797157764435, 0.14292451739311218, 0.10997853428125381, 0.0991024449467659, 0.1065012738108635, 0.12155740708112717, 0.1095810979604721, 0.06964563578367233, 0.08311934769153595, 0.09363401681184769, 0.0941167026758194, 0.08564268052577972, 0.09493856877088547, 0.09063401818275452, 0.09473779052495956, 0.10505300015211105, 0.09355910867452621, 0.09618847072124481, 0.11416846513748169, 0.047205954790115356, 0.053352247923612595, 0.06512638926506042, 0.07829347252845764, 0.05368548259139061, 0.05310077592730522, 0.061235878616571426, 0.07320034503936768, 0.05304349213838577, 0.0639304369688034, 0.07404901832342148, 0.08157160878181458 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 2.7991054736276055e-06, "first_to_last_ratio": 0.33567564773230923, "imbalance_abs_log_ratio": 1.0916099193840287, "last_quartile_mean": 8.338720704159641e-06, "mean": 4.869604932622451e-06, "normalized": [ 1.3821454424103314, 0.49168475606090234, 0.3288222925495297, 0.5565522110146656, 0.3356591845589951, 0.27150745448071056, 0.36651970407492146, 0.8656018713916398, 0.807570094444226, 0.6046893956430209, 0.5574229314033485, 0.5737616390314434, 0.4963818741936961, 0.5307784579393172, 0.634070185437918, 0.6779622813244512, 0.44438775497362176, 0.5832533690857251, 0.5556323701922152, 0.4810431315937026, 1.18029080970422, 1.6827022638352918, 1.8695018571311157, 2.0228435245520657, 2.34814350379181, 2.373785490836653, 2.0489989415300895, 1.3787886304099457, 1.697063360387546, 1.657025164084942, 1.1111991842161706, 1.0842108677157674 ], "peak_layer": 26, "peak_normalized": 2.373785490836653, "population_cv": 0.633958442976093, "rest_mean": 4.129518414528219e-06, "spike_contrast": 2.1469990542091377, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 8.866072130331304e-06, "top_five_layers": [ 26, 25, 27, 24, 23 ], "top_five_spike_overlap": 3 }, "values": [ 6.73050226396299e-06, 2.3943105134094367e-06, 1.6012346577554126e-06, 2.710189392018947e-06, 1.6345276208085124e-06, 1.3221340395830339e-06, 1.7848061588665587e-06, 4.2151391426159535e-06, 3.932547315343982e-06, 2.9445984637277434e-06, 2.714429456318612e-06, 2.793992507577059e-06, 2.4171836230379995e-06, 2.584681396911037e-06, 3.087671302637318e-06, 3.301408469269518e-06, 2.1639928036165657e-06, 2.84021348306851e-06, 2.705710130612715e-06, 2.342490006412845e-06, 5.7475499488646165e-06, 8.194095244107302e-06, 9.103735465032514e-06, 9.850448805082124e-06, 1.1434531188569963e-05, 1.1559397535165772e-05, 9.977815352613106e-06, 6.714155915688025e-06, 8.264028110716026e-06, 8.06905791250756e-06, 5.411101028585108e-06, 5.279678589431569e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 1.1317524240439525e-05, "first_to_last_ratio": 0.339513080994469, "imbalance_abs_log_ratio": 1.080242802550488, "last_quartile_mean": 3.333457493681635e-05, "mean": 1.949880748952637e-05, "normalized": [ 1.4286932908235943, 0.5049056115753852, 0.32964554890894204, 0.5634340555272719, 0.33118891587924415, 0.2710091716518503, 0.3661299679372724, 0.8483642680484571, 0.804718166371966, 0.6029881136144615, 0.5518242753203703, 0.5777754252556628, 0.498611103535574, 0.5310891866013594, 0.6345058048245109, 0.6769952383495429, 0.4484364362023663, 0.587234095622699, 0.5595154795101648, 0.4739030989980023, 1.1834614517077944, 1.6716570379346825, 1.8665741389672614, 2.0107806842727256, 2.3515626041054323, 2.3677397262508313, 2.056023489883435, 1.378063894971612, 1.6924169876959034, 1.6451074971729092, 1.1115744188143237, 1.0740708136643926 ], "peak_layer": 26, "peak_normalized": 2.3677397262508313, "population_cv": 0.6331557978793478, "rest_mean": 1.654940637376125e-05, "spike_contrast": 2.1405948173962637, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.5425573514658026e-05, "top_five_layers": [ 26, 25, 27, 24, 23 ], "top_five_spike_overlap": 3 }, "values": [ 2.7857815439347178e-05, 9.845057320490014e-06, 6.427695097954711e-06, 1.0986292181769386e-05, 6.4577889133943245e-06, 5.284355665935436e-06, 7.1390977609553374e-06, 1.6542091543669812e-05, 1.569104460941162e-05, 1.175754914584104e-05, 1.0759915312519297e-05, 1.1265931789239403e-05, 9.722321919980459e-06, 1.0355605809309054e-05, 1.237210653926013e-05, 1.3200599823903758e-05, 8.743975740799215e-06, 1.1450364581833128e-05, 1.090988462237874e-05, 9.240545296052005e-06, 2.307608701812569e-05, 3.2595318771200255e-05, 3.6395969800651073e-05, 3.920782546629198e-05, 4.5852666517021134e-05, 4.616810110746883e-05, 4.009000622318126e-05, 2.687060259631835e-05, 3.300011303508654e-05, 3.2077634386951104e-05, 2.1674375602742657e-05, 2.094310002576094e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.01003649370977655, "first_to_last_ratio": 0.339513084876601, "imbalance_abs_log_ratio": 1.0802427911160775, "last_quartile_mean": 0.029561434173956513, "mean": 0.017291737414780073, "normalized": [ 1.4286933607192216, 0.5049056305622437, 0.32964555928540584, 0.5634340463604888, 0.33118892727652993, 0.2710091553933497, 0.3661299696274145, 0.8483642135310484, 0.804718172841924, 0.6029881192554497, 0.5518242760175274, 0.5777754038125859, 0.4986110826701228, 0.5310891616614392, 0.6345057774543275, 0.6769952414220057, 0.4484364750027882, 0.5872340836424692, 0.5595154519698056, 0.47390313750915297, 1.1834615019125636, 1.671656984821261, 1.8665741476653213, 2.010780747968474, 2.3515625188363254, 2.367739727742609, 2.0560235043828468, 1.3780639104323187, 1.6924170899382114, 1.6451074252504745, 1.1115743933862032, 1.074070801648091 ], "peak_layer": 26, "peak_normalized": 2.367739727742609, "population_cv": 0.6331557986163227, "rest_mean": 0.014676179029737358, "spike_contrast": 2.1405948122024885, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.031415752694010736, "top_five_layers": [ 26, 25, 27, 24, 23 ], "top_five_spike_overlap": 3 }, "values": [ 0.024704590439796448, 0.008730695582926273, 0.005700144451111555, 0.009742753580212593, 0.005726831965148449, 0.004686219152063131, 0.006331023294478655, 0.0146696912124753, 0.013914975337684155, 0.010426712222397327, 0.009542000479996204, 0.009990740567445755, 0.008621851913630962, 0.00918345432728529, 0.010971707291901112, 0.011706423945724964, 0.007754245772957802, 0.010154297575354576, 0.00967499427497387, 0.008194608613848686, 0.020464105531573296, 0.028905853629112244, 0.03227631002664566, 0.03476989269256592, 0.040662601590156555, 0.04094233363866806, 0.035552218556404114, 0.02382911927998066, 0.02926483191549778, 0.02844676561653614, 0.019221052527427673, 0.018572550266981125 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 6.805292080969139e-07, "first_to_last_ratio": 0.30092757174917156, "imbalance_abs_log_ratio": 1.2008856686062368, "last_quartile_mean": 2.2614385386532376e-06, "mean": 1.2725263340840343e-06, "normalized": [ 1.6834845243340884, 0.43463700446558, 0.3040743266594736, 0.5638769666383798, 0.29234237815578973, 0.23311323047787713, 0.27548471114293566, 0.49127448090743164, 0.672395664421885, 0.5388108554425199, 0.49824103934934877, 0.5359368044531257, 0.4784934412549847, 0.5088767254601788, 0.6481898030543396, 0.6678284516559599, 0.4521033190033952, 0.5648960173878521, 0.5727827724474491, 0.41681060716392426, 1.1691451811952955, 1.7258603826335417, 1.9890952219871774, 2.065244976339804, 2.3584937704988778, 2.405445239475136, 2.2424071255810367, 1.4324114680449918, 1.715637266229551, 1.7139187320414258, 1.1902707513897657, 1.158416760706878 ], "peak_layer": 26, "peak_normalized": 2.405445239475136, "population_cv": 0.6888273490376049, "rest_mean": 1.0694952508132431e-06, "spike_contrast": 2.2149646592119057, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.3688941837463064e-06, "top_five_layers": [ 26, 25, 27, 24, 23 ], "top_five_spike_overlap": 3 }, "values": [ 2.1422783902380615e-06, 5.530870339498506e-07, 3.8694258819305105e-07, 7.175482892307627e-07, 3.720133747719956e-07, 2.9664272460649954e-07, 3.50561549566919e-07, 6.251597142181708e-07, 8.556411899007799e-07, 6.856510026409524e-07, 6.340248432934459e-07, 6.81993697071448e-07, 6.088955046834599e-07, 6.475590339505288e-07, 8.248385938713909e-07, 8.498292913827754e-07, 5.753133791586151e-07, 7.188450581452344e-07, 7.28881161649042e-07, 5.304024739416491e-07, 1.4877680314384634e-06, 2.1962027858535293e-06, 2.5311760509794112e-06, 2.6280786187271588e-06, 3.0012454317329684e-06, 3.0609924124291865e-06, 2.8535221190395532e-06, 1.8227813143312233e-06, 2.183193601013045e-06, 2.181006721002632e-06, 1.5146508758334676e-06, 1.4741158338438254e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 7.5133031316454435e-06, "first_to_last_ratio": 0.34932897048339223, "imbalance_abs_log_ratio": 1.0517411919215074, "last_quartile_mean": 2.1507815745280823e-05, "mean": 1.2924795179003468e-05, "normalized": [ 1.4615313969338561, 0.5162700243303748, 0.25935537284766236, 0.5165777093287385, 0.36393903430003344, 0.2946971906142501, 0.38252345448993275, 0.8555799036464703, 0.8687095561152366, 0.6502467334017257, 0.5976674661296131, 0.6373632289316503, 0.5496099192977181, 0.5791070063152296, 0.6844801639000515, 0.7310121279384592, 0.47269795831053096, 0.6228202309643971, 0.581542274143902, 0.510469291304621, 1.150421437051548, 1.6229460784607708, 1.8343447069374086, 1.9434956522459927, 2.4071782465918914, 2.260495009098695, 1.9496357007805194, 1.3397053074251006, 1.6872360232463604, 1.5881696868538404, 1.0709807944368714, 1.0091913136265487 ], "peak_layer": 25, "peak_normalized": 2.4071782465918914, "population_cv": 0.6062151213661379, "rest_mean": 1.1029931110223751e-05, "spike_contrast": 2.0994746856532434, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.3157061150413938e-05, "top_five_layers": [ 25, 26, 27, 24, 23 ], "top_five_spike_overlap": 3 }, "values": [ 1.8889993953052908e-05, 6.672684321529232e-06, 3.352115072630113e-06, 6.676661087112734e-06, 4.70383747597225e-06, 3.8089008285169257e-06, 4.944037300447235e-06, 1.105819501390215e-05, 1.1227893082832452e-05, 8.404305845033377e-06, 7.724729584879242e-06, 8.237789188569877e-06, 7.103595635271631e-06, 7.484839443350211e-06, 8.84676592249889e-06, 9.448182026972063e-06, 6.109524292696733e-06, 8.049823918554466e-06, 7.516314781241817e-06, 6.597711035283282e-06, 1.4868961443426087e-05, 2.0976245650672354e-05, 2.3708529624855146e-05, 2.5119283236563206e-05, 3.1112285796552896e-05, 2.921643499576021e-05, 2.5198642106261104e-05, 1.7315416698693298e-05, 2.180718001909554e-05, 2.0526767912087962e-05, 1.384220740874298e-05, 1.3043591025052592e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 4.813005659798364e-06, "first_to_last_ratio": 0.3420703192448728, "imbalance_abs_log_ratio": 1.072738951227633, "last_quartile_mean": 1.4070222960071987e-05, "mean": 8.466043212251861e-06, "normalized": [ 1.4298432055714914, 0.507536705786497, 0.21083711328361357, 0.44604949479133454, 0.3830438350309206, 0.31248965486194763, 0.3886666450612197, 0.8695901311801398, 0.9068945859033214, 0.6518204390490815, 0.6079964145186059, 0.6710604616930717, 0.5803243791143339, 0.6219734053285421, 0.7117244218665304, 0.7776793174649611, 0.4984770417537271, 0.6738009638687104, 0.6103175693699558, 0.5530936064574721, 1.0988459758834097, 1.544050813741935, 1.7805735549237978, 1.867632358416996, 2.4573284814864924, 2.1721748792552256, 1.8801553038742365, 1.3874837006768213, 1.6966765883586041, 1.6417164671688658, 1.0895651164770748, 0.9705773677810638 ], "peak_layer": 25, "peak_normalized": 2.4573284814864924, "population_cv": 0.5902940608597187, "rest_mean": 7.290695790558217e-06, "spike_contrast": 2.0317565997721294, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.4812919289397541e-05, "top_five_layers": [ 25, 26, 27, 24, 23 ], "top_five_spike_overlap": 3 }, "values": [ 1.2105114365112968e-05, 4.2968276829924434e-06, 1.7849561118055135e-06, 3.77627429770655e-06, 3.242865659558447e-06, 2.6455509214429185e-06, 3.290468612249242e-06, 7.361987627518829e-06, 7.677808753214777e-06, 5.518340003618505e-06, 5.147323918208713e-06, 5.681226866727229e-06, 4.913051270705182e-06, 5.2656537263828795e-06, 6.02548971073702e-06, 6.583866706932895e-06, 4.220128175802529e-06, 5.704428076569457e-06, 5.1669749154825695e-06, 4.682514372689184e-06, 9.302877515438013e-06, 1.3072000911051873e-05, 1.5074412658577785e-05, 1.5811456250958145e-05, 2.0803849110961892e-05, 1.838972639234271e-05, 1.5917476048343815e-05, 1.1746496966225095e-05, 1.4364137314260006e-05, 1.3898842553317081e-05, 9.224305358657148e-06, 8.216949936468154e-06 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 2.288062012212322e-05, "first_to_last_ratio": 0.31668458159459245, "imbalance_abs_log_ratio": 1.14984901119927, "last_quartile_mean": 7.225050239867414e-05, "mean": 4.142294389453127e-05, "normalized": [ 1.3837145741940322, 0.48463659887438365, 0.2807195438062655, 0.5324742889583569, 0.3306368031599423, 0.26231572683090765, 0.36609177515995783, 0.7783375871874753, 0.8050202038090932, 0.6065175730278692, 0.5435606410521643, 0.5861778688287665, 0.5029476285138275, 0.5253496004159748, 0.642867110675577, 0.6838430609002981, 0.45233066570039904, 0.5927316056714066, 0.5539011375375945, 0.47193711720897, 1.182388275038957, 1.652103590048254, 1.8095144730634385, 2.016165966090558, 2.335040865630095, 2.3625294542173387, 2.138209109497016, 1.4371853733414506, 1.751596732158638, 1.6931232333156794, 1.1284004488716368, 1.1076313672136757 ], "peak_layer": 26, "peak_normalized": 2.3625294542173387, "population_cv": 0.6411220645526207, "rest_mean": 3.5293555339632335e-05, "spike_contrast": 2.111480166105642, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 7.452164209098555e-05, "top_five_layers": [ 26, 25, 27, 24, 23 ], "top_five_spike_overlap": 3 }, "values": [ 5.731753117288463e-05, 2.007507464441005e-05, 1.162822991318535e-05, 2.205665259680245e-05, 1.3695949746761471e-05, 1.0865889635169879e-05, 1.516459906270029e-05, 3.224103420507163e-05, 3.33463067363482e-05, 2.51237433985807e-05, 2.2515881937579252e-05, 2.428121297270991e-05, 2.083357139781583e-05, 2.1761527023045346e-05, 2.6629448257153854e-05, 2.832679274433758e-05, 1.873686778708361e-05, 2.455268804624211e-05, 2.2944215743336827e-05, 1.9549024727893993e-05, 4.8978003178490326e-05, 6.843499431852251e-05, 7.495541649404913e-05, 8.351552969543263e-05, 9.672426676843315e-05, 9.78629250312224e-05, 8.857091597747058e-05, 5.953244908596389e-05, 7.255629316205159e-05, 7.013414870016277e-05, 4.6741668484173715e-05, 4.588135197991505e-05 ] } } } }, "selector": { "expected_uniform_count": 57, "identities_unique": true, "identity_order_sha256": "97fd9721a263024f6a746638f32e0ad9a988cd4c111c00e9e178c1022e5c9d5e", "passed": true, "uniform_count": 57, "uniform_indices": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 56, 57, 58, 59, 60, 61, 62, 63, 64 ], "visit_count": 65, "visits": [ { "branch": "attention", "coefficient": "uniform", "group": 1, "index": 0, "kind": "depth", "layer": 1, "offset": 1, "sources": 1, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 1, "index": 1, "kind": "depth", "layer": 1, "offset": 1, "sources": 2, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 1, "index": 2, "kind": "depth", "layer": 2, "offset": 2, "sources": 2, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 1, "index": 3, "kind": "depth", "layer": 2, "offset": 2, "sources": 2, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 1, "index": 4, "kind": "depth", "layer": 3, "offset": 3, "sources": 2, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 1, "index": 5, "kind": "depth", "layer": 3, "offset": 3, "sources": 2, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 1, "index": 6, "kind": "depth", "layer": 4, "offset": 4, "sources": 2, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 1, "index": 7, "kind": "depth", "layer": 4, "offset": 4, "sources": 2, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 2, "index": 8, "kind": "depth", "layer": 5, "offset": 1, "sources": 2, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 2, "index": 9, "kind": "depth", "layer": 5, "offset": 1, "sources": 3, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 2, "index": 10, "kind": "depth", "layer": 6, "offset": 2, "sources": 3, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 2, "index": 11, "kind": "depth", "layer": 6, "offset": 2, "sources": 3, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 2, "index": 12, "kind": "depth", "layer": 7, "offset": 3, "sources": 3, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 2, "index": 13, "kind": "depth", "layer": 7, "offset": 3, "sources": 3, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 2, "index": 14, "kind": "depth", "layer": 8, "offset": 4, "sources": 3, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 2, "index": 15, "kind": "depth", "layer": 8, "offset": 4, "sources": 3, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 3, "index": 16, "kind": "depth", "layer": 9, "offset": 1, "sources": 3, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 3, "index": 17, "kind": "depth", "layer": 9, "offset": 1, "sources": 4, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 3, "index": 18, "kind": "depth", "layer": 10, "offset": 2, "sources": 4, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 3, "index": 19, "kind": "depth", "layer": 10, "offset": 2, "sources": 4, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 3, "index": 20, "kind": "depth", "layer": 11, "offset": 3, "sources": 4, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 3, "index": 21, "kind": "depth", "layer": 11, "offset": 3, "sources": 4, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 3, "index": 22, "kind": "depth", "layer": 12, "offset": 4, "sources": 4, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 3, "index": 23, "kind": "depth", "layer": 12, "offset": 4, "sources": 4, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 4, "index": 24, "kind": "depth", "layer": 13, "offset": 1, "sources": 4, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 4, "index": 25, "kind": "depth", "layer": 13, "offset": 1, "sources": 5, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 4, "index": 26, "kind": "depth", "layer": 14, "offset": 2, "sources": 5, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 4, "index": 27, "kind": "depth", "layer": 14, "offset": 2, "sources": 5, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 4, "index": 28, "kind": "depth", "layer": 15, "offset": 3, "sources": 5, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 4, "index": 29, "kind": "depth", "layer": 15, "offset": 3, "sources": 5, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 4, "index": 30, "kind": "depth", "layer": 16, "offset": 4, "sources": 5, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 4, "index": 31, "kind": "depth", "layer": 16, "offset": 4, "sources": 5, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 5, "index": 32, "kind": "depth", "layer": 17, "offset": 1, "sources": 5, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 5, "index": 33, "kind": "depth", "layer": 17, "offset": 1, "sources": 6, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 5, "index": 34, "kind": "depth", "layer": 18, "offset": 2, "sources": 6, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 5, "index": 35, "kind": "depth", "layer": 18, "offset": 2, "sources": 6, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 5, "index": 36, "kind": "depth", "layer": 19, "offset": 3, "sources": 6, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 5, "index": 37, "kind": "depth", "layer": 19, "offset": 3, "sources": 6, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 5, "index": 38, "kind": "depth", "layer": 20, "offset": 4, "sources": 6, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 5, "index": 39, "kind": "depth", "layer": 20, "offset": 4, "sources": 6, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 6, "index": 40, "kind": "depth", "layer": 21, "offset": 1, "sources": 6, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 6, "index": 41, "kind": "depth", "layer": 21, "offset": 1, "sources": 7, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 6, "index": 42, "kind": "depth", "layer": 22, "offset": 2, "sources": 7, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 6, "index": 43, "kind": "depth", "layer": 22, "offset": 2, "sources": 7, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 6, "index": 44, "kind": "depth", "layer": 23, "offset": 3, "sources": 7, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 6, "index": 45, "kind": "depth", "layer": 23, "offset": 3, "sources": 7, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 6, "index": 46, "kind": "depth", "layer": 24, "offset": 4, "sources": 7, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 6, "index": 47, "kind": "depth", "layer": 24, "offset": 4, "sources": 7, "use_uniform": true }, { "branch": "attention", "coefficient": "detached_learned", "group": 7, "index": 48, "kind": "depth", "layer": 25, "offset": 1, "sources": 7, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 7, "index": 49, "kind": "depth", "layer": 25, "offset": 1, "sources": 8, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 7, "index": 50, "kind": "depth", "layer": 26, "offset": 2, "sources": 8, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 7, "index": 51, "kind": "depth", "layer": 26, "offset": 2, "sources": 8, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 7, "index": 52, "kind": "depth", "layer": 27, "offset": 3, "sources": 8, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 7, "index": 53, "kind": "depth", "layer": 27, "offset": 3, "sources": 8, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 7, "index": 54, "kind": "depth", "layer": 28, "offset": 4, "sources": 8, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 7, "index": 55, "kind": "depth", "layer": 28, "offset": 4, "sources": 8, "use_uniform": false }, { "branch": "attention", "coefficient": "uniform", "group": 8, "index": 56, "kind": "depth", "layer": 29, "offset": 1, "sources": 8, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 8, "index": 57, "kind": "depth", "layer": 29, "offset": 1, "sources": 9, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 8, "index": 58, "kind": "depth", "layer": 30, "offset": 2, "sources": 9, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 8, "index": 59, "kind": "depth", "layer": 30, "offset": 2, "sources": 9, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 8, "index": 60, "kind": "depth", "layer": 31, "offset": 3, "sources": 9, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 8, "index": 61, "kind": "depth", "layer": 31, "offset": 3, "sources": 9, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 8, "index": 62, "kind": "depth", "layer": 32, "offset": 4, "sources": 9, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 8, "index": 63, "kind": "depth", "layer": 32, "offset": 4, "sources": 9, "use_uniform": true }, { "branch": "output", "coefficient": "uniform", "group": null, "index": 64, "kind": "output", "layer": null, "offset": null, "sources": 9, "use_uniform": true } ], "visits_sha256": "9dc56a1d63831ead7802d3ab462a452e1dc050e7e8c2166997eba292fe80e060" } }, "uniform_except_groups_6_7": { "bits_per_byte": 1.8740129107275643, "forward": { "activation_sha256": { "attention_branch_output": "5e6ecbcd78543fc42d4098a51169dca446d404ce384015643e84d2c78433c949", "mlp_branch_output": "c0b8018c349b1bd09e5d17858eb312a2323da0e92de9651e2bb08ae4adf115c7", "post_attention_state": "90a633f12d984935028fe5d7212be48e7da7b4a0806751939a551938d16dc540", "post_mlp_state": "89f2226456bfe96e245d2aad0f699d32c676f9f53099789cd870253d002402f5", "pre_attention_input": "ee695e78937de01b7128d3866b8fc5dd328ebdf2c159be37918d4f02952d5248", "pre_mlp_input": "e53fa425d89dfeb0e9395eca9bcd73dc0a55314abf45c5b10b43e2cbfa724345" }, "logits_sha256": "bdacba5675f0579cec629bbf01b36c8660ad576cfdf9fef1bea8259eaeac9ec2", "loss_nats": 1.2989667654037476, "mixer_summary_sha256": "eb3ac0f07de9b755d6e67a510ab68d09d9ad666bb7b5d020ba7d8c5055c399b2" }, "loss_nats": 1.2989667654037476, "loss_scale": 1.0, "mixers": [ { "branch": "attention", "entropy_mean": -0.0, "group": 1, "latest_source_mass": 1.0, "layer": 1, "max_source_mass": 1.0, "mixer_index": 0, "normalized_entropy": 1.0, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 1.0, "median": 1.0, "p05": 1.0, "p95": 1.0 } ], "sources": 1 }, { "branch": "mlp", "entropy_mean": 0.22886337339878082, "group": 1, "latest_source_mass": 0.06350618600845337, "layer": 1, "max_source_mass": 0.9364938139915466, "mixer_index": 1, "normalized_entropy": 0.3301800538435402, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.9364938139915466, "median": 0.9452826976776123, "p05": 0.8725647330284119, "p95": 0.9712384939193726 }, { "label": "current_group_partial", "mean": 0.06350618600845337, "median": 0.0547172985970974, "p05": 0.028761589899659157, "p95": 0.12743529677391052 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.603729784488678, "group": 1, "latest_source_mass": 0.4133155345916748, "layer": 2, "max_source_mass": 0.5866844654083252, "mixer_index": 2, "normalized_entropy": 0.8709979661187783, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.5866844654083252, "median": 0.5734395384788513, "p05": 0.29813364148139954, "p95": 0.8843642473220825 }, { "label": "current_group_partial", "mean": 0.4133155345916748, "median": 0.4265604615211487, "p05": 0.11563576757907867, "p95": 0.7018663287162781 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.3967982828617096, "group": 1, "latest_source_mass": 0.14110636711120605, "layer": 2, "max_source_mass": 0.858893632888794, "mixer_index": 3, "normalized_entropy": 0.5724589149178446, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.858893632888794, "median": 0.8705973029136658, "p05": 0.7599387168884277, "p95": 0.9270997643470764 }, { "label": "current_group_partial", "mean": 0.14110636711120605, "median": 0.12940272688865662, "p05": 0.07290027290582657, "p95": 0.2400612235069275 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6852496862411499, "group": 1, "latest_source_mass": 0.518975019454956, "layer": 3, "max_source_mass": 0.518975019454956, "mixer_index": 4, "normalized_entropy": 0.9886063241108252, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.48102498054504395, "median": 0.48102718591690063, "p05": 0.38786420226097107, "p95": 0.580143928527832 }, { "label": "current_group_partial", "mean": 0.518975019454956, "median": 0.5189728140830994, "p05": 0.41985607147216797, "p95": 0.6121357679367065 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.5890734195709229, "group": 1, "latest_source_mass": 0.3123423159122467, "layer": 3, "max_source_mass": 0.6876576542854309, "mixer_index": 5, "normalized_entropy": 0.8498533011344741, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.6876576542854309, "median": 0.6909762620925903, "p05": 0.48803940415382385, "p95": 0.8900076746940613 }, { "label": "current_group_partial", "mean": 0.3123423159122467, "median": 0.3090237081050873, "p05": 0.10999235510826111, "p95": 0.5119606256484985 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6873323321342468, "group": 1, "latest_source_mass": 0.4831485152244568, "layer": 4, "max_source_mass": 0.5168514251708984, "mixer_index": 6, "normalized_entropy": 0.9916109470127238, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.5168514251708984, "median": 0.5219584703445435, "p05": 0.4251725673675537, "p95": 0.5947866439819336 }, { "label": "current_group_partial", "mean": 0.4831485152244568, "median": 0.47804147005081177, "p05": 0.405213326215744, "p95": 0.5748274326324463 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.487301766872406, "group": 1, "latest_source_mass": 0.2060384750366211, "layer": 4, "max_source_mass": 0.7939615249633789, "mixer_index": 7, "normalized_entropy": 0.7030278424832499, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.7939615249633789, "median": 0.7953404188156128, "p05": 0.6590759754180908, "p95": 0.9329437017440796 }, { "label": "current_group_partial", "mean": 0.2060384750366211, "median": 0.20465964078903198, "p05": 0.06705628335475922, "p95": 0.3409239649772644 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.674697756767273, "group": 2, "latest_source_mass": 0.42952626943588257, "layer": 5, "max_source_mass": 0.5704737305641174, "mixer_index": 8, "normalized_entropy": 0.9733831077870527, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.5704737305641174, "median": 0.5696132779121399, "p05": 0.4695202708244324, "p95": 0.6750621199607849 }, { "label": "completed_group_1", "mean": 0.42952626943588257, "median": 0.4303867816925049, "p05": 0.3249378800392151, "p95": 0.5304797887802124 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.9691112041473389, "group": 2, "latest_source_mass": 0.2681158185005188, "layer": 5, "max_source_mass": 0.5506307482719421, "mixer_index": 9, "normalized_entropy": 0.8821230329784768, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.5506307482719421, "median": 0.543993353843689, "p05": 0.41262108087539673, "p95": 0.7161091566085815 }, { "label": "completed_group_1", "mean": 0.18125343322753906, "median": 0.1794438362121582, "p05": 0.11575750261545181, "p95": 0.25031501054763794 }, { "label": "current_group_partial", "mean": 0.2681158185005188, "median": 0.2664111256599426, "p05": 0.1564716249704361, "p95": 0.3817881643772125 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0745112895965576, "group": 2, "latest_source_mass": 0.412794291973114, "layer": 6, "max_source_mass": 0.412794291973114, "mixer_index": 10, "normalized_entropy": 0.9780623252441762, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.34262996912002563, "median": 0.3412924110889435, "p05": 0.29556700587272644, "p95": 0.39125555753707886 }, { "label": "completed_group_1", "mean": 0.24457570910453796, "median": 0.24382859468460083, "p05": 0.22120657563209534, "p95": 0.2734297513961792 }, { "label": "current_group_partial", "mean": 0.412794291973114, "median": 0.4151499271392822, "p05": 0.37735897302627563, "p95": 0.43749260902404785 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0074045658111572, "group": 2, "latest_source_mass": 0.3775783181190491, "layer": 6, "max_source_mass": 0.4399106502532959, "mixer_index": 11, "normalized_entropy": 0.9169791528842925, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.4399106502532959, "median": 0.44856905937194824, "p05": 0.22063353657722473, "p95": 0.6359403133392334 }, { "label": "completed_group_1", "mean": 0.18251106142997742, "median": 0.18562838435173035, "p05": 0.1196163073182106, "p95": 0.23045453429222107 }, { "label": "current_group_partial", "mean": 0.3775783181190491, "median": 0.3675365447998047, "p05": 0.23817461729049683, "p95": 0.5557451844215393 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.060497522354126, "group": 2, "latest_source_mass": 0.4025024175643921, "layer": 7, "max_source_mass": 0.4025024175643921, "mixer_index": 12, "normalized_entropy": 0.9653064445872968, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.3786153197288513, "median": 0.3907395303249359, "p05": 0.28039106726646423, "p95": 0.4436405599117279 }, { "label": "completed_group_1", "mean": 0.2188822478055954, "median": 0.21226292848587036, "p05": 0.1885872334241867, "p95": 0.2672030031681061 }, { "label": "current_group_partial", "mean": 0.4025024175643921, "median": 0.39815032482147217, "p05": 0.3533596992492676, "p95": 0.460100919008255 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 0.9941019415855408, "group": 2, "latest_source_mass": 0.4801652431488037, "layer": 7, "max_source_mass": 0.4801652431488037, "mixer_index": 13, "normalized_entropy": 0.9048705824970601, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.32296955585479736, "median": 0.2803739905357361, "p05": 0.1784922033548355, "p95": 0.6440245509147644 }, { "label": "completed_group_1", "mean": 0.19686520099639893, "median": 0.2053825855255127, "p05": 0.09813719242811203, "p95": 0.2504696547985077 }, { "label": "current_group_partial", "mean": 0.4801652431488037, "median": 0.5057140588760376, "p05": 0.2529003322124481, "p95": 0.598870575428009 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0503522157669067, "group": 2, "latest_source_mass": 0.4001518487930298, "layer": 8, "max_source_mass": 0.40130966901779175, "mixer_index": 14, "normalized_entropy": 0.9560717885654542, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.40130966901779175, "median": 0.396759569644928, "p05": 0.3479611873626709, "p95": 0.4733521044254303 }, { "label": "completed_group_1", "mean": 0.19853851199150085, "median": 0.19769755005836487, "p05": 0.17136667668819427, "p95": 0.22788912057876587 }, { "label": "current_group_partial", "mean": 0.4001518487930298, "median": 0.40291696786880493, "p05": 0.3506356179714203, "p95": 0.43941178917884827 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0129402875900269, "group": 2, "latest_source_mass": 0.42161279916763306, "layer": 8, "max_source_mass": 0.42161279916763306, "mixer_index": 15, "normalized_entropy": 0.9220179839951123, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.4093267023563385, "median": 0.4046899676322937, "p05": 0.2895161807537079, "p95": 0.5762131810188293 }, { "label": "completed_group_1", "mean": 0.16906049847602844, "median": 0.16888955235481262, "p05": 0.11233551055192947, "p95": 0.22477403283119202 }, { "label": "current_group_partial", "mean": 0.42161279916763306, "median": 0.4227708578109741, "p05": 0.3131049573421478, "p95": 0.519075334072113 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0774019956588745, "group": 3, "latest_source_mass": 0.3560434877872467, "layer": 9, "max_source_mass": 0.38510820269584656, "mixer_index": 16, "normalized_entropy": 0.9806935592947451, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.38510820269584656, "median": 0.3892287313938141, "p05": 0.2934730052947998, "p95": 0.4718902111053467 }, { "label": "completed_group_1", "mean": 0.25884830951690674, "median": 0.25734007358551025, "p05": 0.22217319905757904, "p95": 0.29551801085472107 }, { "label": "completed_group_2", "mean": 0.3560434877872467, "median": 0.35412120819091797, "p05": 0.29444968700408936, "p95": 0.41908568143844604 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.2740377187728882, "group": 3, "latest_source_mass": 0.3633861243724823, "layer": 9, "max_source_mass": 0.3633861243724823, "mixer_index": 17, "normalized_entropy": 0.9190239493895668, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.29550763964653015, "median": 0.30360478162765503, "p05": 0.16904757916927338, "p95": 0.45146965980529785 }, { "label": "completed_group_1", "mean": 0.09541244804859161, "median": 0.09486871212720871, "p05": 0.061360105872154236, "p95": 0.13037073612213135 }, { "label": "completed_group_2", "mean": 0.24569380283355713, "median": 0.24333015084266663, "p05": 0.1698218435049057, "p95": 0.31589236855506897 }, { "label": "current_group_partial", "mean": 0.3633861243724823, "median": 0.36472851037979126, "p05": 0.2795379161834717, "p95": 0.4399876892566681 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.360302448272705, "group": 3, "latest_source_mass": 0.28569501638412476, "layer": 10, "max_source_mass": 0.3109527826309204, "mixer_index": 18, "normalized_entropy": 0.9812507981160736, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.3109527826309204, "median": 0.32172977924346924, "p05": 0.23520725965499878, "p95": 0.37052470445632935 }, { "label": "completed_group_1", "mean": 0.20984819531440735, "median": 0.20958667993545532, "p05": 0.1891367882490158, "p95": 0.23184512555599213 }, { "label": "completed_group_2", "mean": 0.19350400567054749, "median": 0.19153150916099548, "p05": 0.16607457399368286, "p95": 0.22877082228660583 }, { "label": "current_group_partial", "mean": 0.28569501638412476, "median": 0.28060320019721985, "p05": 0.23824527859687805, "p95": 0.3439165949821472 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.2722704410552979, "group": 3, "latest_source_mass": 0.29912811517715454, "layer": 10, "max_source_mass": 0.35036465525627136, "mixer_index": 19, "normalized_entropy": 0.9177491279900463, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.35036465525627136, "median": 0.36210548877716064, "p05": 0.12807878851890564, "p95": 0.5797606706619263 }, { "label": "completed_group_1", "mean": 0.12631990015506744, "median": 0.1264108121395111, "p05": 0.06977735459804535, "p95": 0.1755298376083374 }, { "label": "completed_group_2", "mean": 0.22418735921382904, "median": 0.22563719749450684, "p05": 0.1176326721906662, "p95": 0.32001155614852905 }, { "label": "current_group_partial", "mean": 0.29912811517715454, "median": 0.28589946031570435, "p05": 0.193109393119812, "p95": 0.44119665026664734 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3558036088943481, "group": 3, "latest_source_mass": 0.314197838306427, "layer": 11, "max_source_mass": 0.314197838306427, "mixer_index": 20, "normalized_entropy": 0.978005571485618, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.298976868391037, "median": 0.2961939573287964, "p05": 0.24950163066387177, "p95": 0.35835936665534973 }, { "label": "completed_group_1", "mean": 0.1941717267036438, "median": 0.1918032318353653, "p05": 0.16809208691120148, "p95": 0.22895419597625732 }, { "label": "completed_group_2", "mean": 0.1926535815000534, "median": 0.19306811690330505, "p05": 0.16983944177627563, "p95": 0.2155935913324356 }, { "label": "current_group_partial", "mean": 0.314197838306427, "median": 0.31541913747787476, "p05": 0.2673022747039795, "p95": 0.35941022634506226 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.2696614265441895, "group": 3, "latest_source_mass": 0.2862822115421295, "layer": 11, "max_source_mass": 0.3667730689048767, "mixer_index": 21, "normalized_entropy": 0.9158671218416545, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.3667730689048767, "median": 0.4067659378051758, "p05": 0.10593366622924805, "p95": 0.5672274827957153 }, { "label": "completed_group_1", "mean": 0.15310388803482056, "median": 0.15050218999385834, "p05": 0.08651046454906464, "p95": 0.22459912300109863 }, { "label": "completed_group_2", "mean": 0.19384080171585083, "median": 0.18732509016990662, "p05": 0.1267009675502777, "p95": 0.281107634305954 }, { "label": "current_group_partial", "mean": 0.2862822115421295, "median": 0.26271265745162964, "p05": 0.17024393379688263, "p95": 0.475044310092926 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3606936931610107, "group": 3, "latest_source_mass": 0.3028184771537781, "layer": 12, "max_source_mass": 0.3028184771537781, "mixer_index": 22, "normalized_entropy": 0.9815330216461395, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.298227459192276, "median": 0.2954302132129669, "p05": 0.25287362933158875, "p95": 0.35405808687210083 }, { "label": "completed_group_1", "mean": 0.18611131608486176, "median": 0.1850002110004425, "p05": 0.15901757776737213, "p95": 0.21725836396217346 }, { "label": "completed_group_2", "mean": 0.21284276247024536, "median": 0.21402311325073242, "p05": 0.18819917738437653, "p95": 0.23418807983398438 }, { "label": "current_group_partial", "mean": 0.3028184771537781, "median": 0.3041122555732727, "p05": 0.25661808252334595, "p95": 0.34458184242248535 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.269316554069519, "group": 3, "latest_source_mass": 0.2653314769268036, "layer": 12, "max_source_mass": 0.4081852436065674, "mixer_index": 23, "normalized_entropy": 0.9156183489371815, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.4081852436065674, "median": 0.4254342317581177, "p05": 0.20765496790409088, "p95": 0.587238073348999 }, { "label": "completed_group_1", "mean": 0.14420127868652344, "median": 0.1438048928976059, "p05": 0.1064920499920845, "p95": 0.1871720254421234 }, { "label": "completed_group_2", "mean": 0.18228203058242798, "median": 0.17507661879062653, "p05": 0.12183455377817154, "p95": 0.26505547761917114 }, { "label": "current_group_partial", "mean": 0.2653314769268036, "median": 0.24924281239509583, "p05": 0.15609969198703766, "p95": 0.4061311185359955 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3636136054992676, "group": 4, "latest_source_mass": 0.3136237859725952, "layer": 13, "max_source_mass": 0.3136237859725952, "mixer_index": 24, "normalized_entropy": 0.9836392931712563, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.27427661418914795, "median": 0.27403637766838074, "p05": 0.22928935289382935, "p95": 0.319773405790329 }, { "label": "completed_group_1", "mean": 0.21341991424560547, "median": 0.20697325468063354, "p05": 0.18552586436271667, "p95": 0.26688331365585327 }, { "label": "completed_group_2", "mean": 0.19867971539497375, "median": 0.197545126080513, "p05": 0.1794913411140442, "p95": 0.222100168466568 }, { "label": "completed_group_3", "mean": 0.3136237859725952, "median": 0.31875866651535034, "p05": 0.24156072735786438, "p95": 0.3695686459541321 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.570773959159851, "group": 4, "latest_source_mass": 0.2294243425130844, "layer": 13, "max_source_mass": 0.2658740282058716, "mixer_index": 25, "normalized_entropy": 0.9759767351225285, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.2658740282058716, "median": 0.2747163474559784, "p05": 0.17927880585193634, "p95": 0.3512483835220337 }, { "label": "completed_group_1", "mean": 0.13419361412525177, "median": 0.13374894857406616, "p05": 0.11072897166013718, "p95": 0.1586270034313202 }, { "label": "completed_group_2", "mean": 0.1907704919576645, "median": 0.19074805080890656, "p05": 0.14699944853782654, "p95": 0.23316305875778198 }, { "label": "completed_group_3", "mean": 0.17973749339580536, "median": 0.17616114020347595, "p05": 0.1363743096590042, "p95": 0.23202598094940186 }, { "label": "current_group_partial", "mean": 0.2294243425130844, "median": 0.22995705902576447, "p05": 0.17975632846355438, "p95": 0.2789157032966614 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.5560038089752197, "group": 4, "latest_source_mass": 0.32468360662460327, "layer": 14, "max_source_mass": 0.32468360662460327, "mixer_index": 26, "normalized_entropy": 0.9667995248241249, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.20477107167243958, "median": 0.206403911113739, "p05": 0.16619175672531128, "p95": 0.2431035190820694 }, { "label": "completed_group_1", "mean": 0.15721669793128967, "median": 0.15658551454544067, "p05": 0.13878680765628815, "p95": 0.17778633534908295 }, { "label": "completed_group_2", "mean": 0.13811248540878296, "median": 0.13743364810943604, "p05": 0.12352626770734787, "p95": 0.15553249418735504 }, { "label": "completed_group_3", "mean": 0.1752161681652069, "median": 0.17309701442718506, "p05": 0.15223677456378937, "p95": 0.20474471151828766 }, { "label": "current_group_partial", "mean": 0.32468360662460327, "median": 0.32497870922088623, "p05": 0.2946181893348694, "p95": 0.35203880071640015 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.5659973621368408, "group": 4, "latest_source_mass": 0.26703086495399475, "layer": 14, "max_source_mass": 0.26703086495399475, "mixer_index": 27, "normalized_entropy": 0.9730088685238187, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.20531430840492249, "median": 0.20229148864746094, "p05": 0.10698921978473663, "p95": 0.3393132984638214 }, { "label": "completed_group_1", "mean": 0.15986156463623047, "median": 0.15952220559120178, "p05": 0.11969029903411865, "p95": 0.20180393755435944 }, { "label": "completed_group_2", "mean": 0.21557100117206573, "median": 0.21654027700424194, "p05": 0.1673085242509842, "p95": 0.2610440254211426 }, { "label": "completed_group_3", "mean": 0.15222224593162537, "median": 0.1510905921459198, "p05": 0.11800260841846466, "p95": 0.18893754482269287 }, { "label": "current_group_partial", "mean": 0.26703086495399475, "median": 0.258539617061615, "p05": 0.19330690801143646, "p95": 0.36521556973457336 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.5552496910095215, "group": 4, "latest_source_mass": 0.31774693727493286, "layer": 15, "max_source_mass": 0.31774693727493286, "mixer_index": 28, "normalized_entropy": 0.9663309649872576, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.2162160575389862, "median": 0.2145705670118332, "p05": 0.17727568745613098, "p95": 0.26878347992897034 }, { "label": "completed_group_1", "mean": 0.17982497811317444, "median": 0.18024154007434845, "p05": 0.15453973412513733, "p95": 0.2030555009841919 }, { "label": "completed_group_2", "mean": 0.1307673156261444, "median": 0.13107559084892273, "p05": 0.11463287472724915, "p95": 0.14630235731601715 }, { "label": "completed_group_3", "mean": 0.1554446816444397, "median": 0.15516987442970276, "p05": 0.13627474009990692, "p95": 0.1754152774810791 }, { "label": "current_group_partial", "mean": 0.31774693727493286, "median": 0.319039523601532, "p05": 0.27606821060180664, "p95": 0.35650834441185 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.5068919658660889, "group": 4, "latest_source_mass": 0.3298327624797821, "layer": 15, "max_source_mass": 0.3298327624797821, "mixer_index": 29, "normalized_entropy": 0.9362846209998111, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.20646697282791138, "median": 0.18775638937950134, "p05": 0.0806753933429718, "p95": 0.4274584949016571 }, { "label": "completed_group_1", "mean": 0.1197461187839508, "median": 0.11989086866378784, "p05": 0.0809398666024208, "p95": 0.15743419528007507 }, { "label": "completed_group_2", "mean": 0.19215428829193115, "median": 0.19620640575885773, "p05": 0.1148109883069992, "p95": 0.2491212636232376 }, { "label": "completed_group_3", "mean": 0.15179988741874695, "median": 0.15212899446487427, "p05": 0.10697861760854721, "p95": 0.19542087614536285 }, { "label": "current_group_partial", "mean": 0.3298327624797821, "median": 0.32837340235710144, "p05": 0.22746729850769043, "p95": 0.4339301884174347 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.5435223579406738, "group": 4, "latest_source_mass": 0.3224271535873413, "layer": 16, "max_source_mass": 0.3224271535873413, "mixer_index": 30, "normalized_entropy": 0.9590443632623663, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.22410975396633148, "median": 0.22760415077209473, "p05": 0.14593973755836487, "p95": 0.30093055963516235 }, { "label": "completed_group_1", "mean": 0.16986176371574402, "median": 0.16963240504264832, "p05": 0.13569067418575287, "p95": 0.20651353895664215 }, { "label": "completed_group_2", "mean": 0.13080403208732605, "median": 0.13075244426727295, "p05": 0.10945598781108856, "p95": 0.15187400579452515 }, { "label": "completed_group_3", "mean": 0.15279728174209595, "median": 0.15036612749099731, "p05": 0.12810495495796204, "p95": 0.18468116223812103 }, { "label": "current_group_partial", "mean": 0.3224271535873413, "median": 0.3164973258972168, "p05": 0.2600978910923004, "p95": 0.40530598163604736 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.4949920177459717, "group": 4, "latest_source_mass": 0.31847020983695984, "layer": 16, "max_source_mass": 0.31847020983695984, "mixer_index": 31, "normalized_entropy": 0.9288907675133354, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.27174079418182373, "median": 0.2585451304912567, "p05": 0.13639721274375916, "p95": 0.4698348045349121 }, { "label": "completed_group_1", "mean": 0.12178683280944824, "median": 0.11714950203895569, "p05": 0.08594125509262085, "p95": 0.1723237782716751 }, { "label": "completed_group_2", "mean": 0.1529826819896698, "median": 0.15223106741905212, "p05": 0.10371473431587219, "p95": 0.203575000166893 }, { "label": "completed_group_3", "mean": 0.1350194811820984, "median": 0.13363957405090332, "p05": 0.08964549005031586, "p95": 0.18468207120895386 }, { "label": "current_group_partial", "mean": 0.31847020983695984, "median": 0.3185216188430786, "p05": 0.21444711089134216, "p95": 0.4301546812057495 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.544264793395996, "group": 5, "latest_source_mass": 0.2861037850379944, "layer": 17, "max_source_mass": 0.2861037850379944, "mixer_index": 32, "normalized_entropy": 0.9595056643474137, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.24016115069389343, "median": 0.23868846893310547, "p05": 0.18435582518577576, "p95": 0.2983453571796417 }, { "label": "completed_group_1", "mean": 0.20730282366275787, "median": 0.1998077780008316, "p05": 0.16255050897598267, "p95": 0.28107208013534546 }, { "label": "completed_group_2", "mean": 0.09692573547363281, "median": 0.0967404842376709, "p05": 0.08098706603050232, "p95": 0.11221462488174438 }, { "label": "completed_group_3", "mean": 0.1695065200328827, "median": 0.1696585714817047, "p05": 0.136396586894989, "p95": 0.20130634307861328 }, { "label": "completed_group_4", "mean": 0.2861037850379944, "median": 0.28883200883865356, "p05": 0.22401469945907593, "p95": 0.3409370183944702 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.686682939529419, "group": 5, "latest_source_mass": 0.12281867861747742, "layer": 17, "max_source_mass": 0.2885604202747345, "mixer_index": 33, "normalized_entropy": 0.9413556721740635, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.16755425930023193, "median": 0.1327776312828064, "p05": 0.0790192037820816, "p95": 0.3667412996292114 }, { "label": "completed_group_1", "mean": 0.10962115228176117, "median": 0.10525335371494293, "p05": 0.07840172946453094, "p95": 0.15405339002609253 }, { "label": "completed_group_2", "mean": 0.1783454418182373, "median": 0.17896884679794312, "p05": 0.10802385956048965, "p95": 0.24048911035060883 }, { "label": "completed_group_3", "mean": 0.13310004770755768, "median": 0.13033035397529602, "p05": 0.0904831737279892, "p95": 0.17934636771678925 }, { "label": "completed_group_4", "mean": 0.2885604202747345, "median": 0.2831825315952301, "p05": 0.16967125236988068, "p95": 0.4189499616622925 }, { "label": "current_group_partial", "mean": 0.12281867861747742, "median": 0.12062528729438782, "p05": 0.07461707293987274, "p95": 0.1782219111919403 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.7070834636688232, "group": 5, "latest_source_mass": 0.2988377809524536, "layer": 18, "max_source_mass": 0.2988377809524536, "mixer_index": 34, "normalized_entropy": 0.9527414214834803, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.181177020072937, "median": 0.18419909477233887, "p05": 0.13624237477779388, "p95": 0.22192169725894928 }, { "label": "completed_group_1", "mean": 0.14557164907455444, "median": 0.144476979970932, "p05": 0.1293315887451172, "p95": 0.16486899554729462 }, { "label": "completed_group_2", "mean": 0.08318156749010086, "median": 0.0820261612534523, "p05": 0.07088497281074524, "p95": 0.0990293100476265 }, { "label": "completed_group_3", "mean": 0.11424209177494049, "median": 0.11267061531543732, "p05": 0.0979941189289093, "p95": 0.1353992223739624 }, { "label": "completed_group_4", "mean": 0.17698991298675537, "median": 0.17597860097885132, "p05": 0.14837387204170227, "p95": 0.21050144731998444 }, { "label": "current_group_partial", "mean": 0.2988377809524536, "median": 0.2989804744720459, "p05": 0.2721370458602905, "p95": 0.32572072744369507 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.7419235706329346, "group": 5, "latest_source_mass": 0.26016175746917725, "layer": 18, "max_source_mass": 0.26016175746917725, "mixer_index": 35, "normalized_entropy": 0.972186055410333, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1532188355922699, "median": 0.1451343297958374, "p05": 0.11514341086149216, "p95": 0.22103679180145264 }, { "label": "completed_group_1", "mean": 0.11420352011919022, "median": 0.11261284351348877, "p05": 0.0928458571434021, "p95": 0.14123988151550293 }, { "label": "completed_group_2", "mean": 0.16597424447536469, "median": 0.1655328869819641, "p05": 0.13871800899505615, "p95": 0.19611573219299316 }, { "label": "completed_group_3", "mean": 0.1250365972518921, "median": 0.12369821965694427, "p05": 0.10375828295946121, "p95": 0.14912742376327515 }, { "label": "completed_group_4", "mean": 0.18140508234500885, "median": 0.1796584278345108, "p05": 0.14511024951934814, "p95": 0.22286373376846313 }, { "label": "current_group_partial", "mean": 0.26016175746917725, "median": 0.26266711950302124, "p05": 0.1889389455318451, "p95": 0.3234381675720215 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.7127439975738525, "group": 5, "latest_source_mass": 0.2907913327217102, "layer": 19, "max_source_mass": 0.2907913327217102, "mixer_index": 36, "normalized_entropy": 0.9559006256078307, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.18731768429279327, "median": 0.19042012095451355, "p05": 0.13879702985286713, "p95": 0.23792000114917755 }, { "label": "completed_group_1", "mean": 0.16715484857559204, "median": 0.1667211651802063, "p05": 0.14574480056762695, "p95": 0.19026139378547668 }, { "label": "completed_group_2", "mean": 0.08496320247650146, "median": 0.08452782034873962, "p05": 0.07410907745361328, "p95": 0.097386933863163 }, { "label": "completed_group_3", "mean": 0.11362075805664062, "median": 0.11248823255300522, "p05": 0.0977015346288681, "p95": 0.13219426572322845 }, { "label": "completed_group_4", "mean": 0.15615218877792358, "median": 0.15540224313735962, "p05": 0.1289147436618805, "p95": 0.18598109483718872 }, { "label": "current_group_partial", "mean": 0.2907913327217102, "median": 0.2900415062904358, "p05": 0.25822532176971436, "p95": 0.32475030422210693 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.6076834201812744, "group": 5, "latest_source_mass": 0.3738824725151062, "layer": 19, "max_source_mass": 0.3738824725151062, "mixer_index": 37, "normalized_entropy": 0.8972652009334232, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.15032559633255005, "median": 0.12544110417366028, "p05": 0.06976518034934998, "p95": 0.35840943455696106 }, { "label": "completed_group_1", "mean": 0.08624927699565887, "median": 0.08602266013622284, "p05": 0.061998721212148666, "p95": 0.11138717085123062 }, { "label": "completed_group_2", "mean": 0.15736976265907288, "median": 0.15802210569381714, "p05": 0.10121746361255646, "p95": 0.20687000453472137 }, { "label": "completed_group_3", "mean": 0.08584291487932205, "median": 0.0827617198228836, "p05": 0.060561805963516235, "p95": 0.11351694166660309 }, { "label": "completed_group_4", "mean": 0.14632999897003174, "median": 0.1426895558834076, "p05": 0.10106660425662994, "p95": 0.20179018378257751 }, { "label": "current_group_partial", "mean": 0.3738824725151062, "median": 0.3799295425415039, "p05": 0.23631010949611664, "p95": 0.4884527325630188 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.709290862083435, "group": 5, "latest_source_mass": 0.2901708781719208, "layer": 20, "max_source_mass": 0.2901708781719208, "mixer_index": 38, "normalized_entropy": 0.9539733939957076, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.18991519510746002, "median": 0.1924753338098526, "p05": 0.14201581478118896, "p95": 0.23709386587142944 }, { "label": "completed_group_1", "mean": 0.17091524600982666, "median": 0.17006167769432068, "p05": 0.15297070145606995, "p95": 0.19153237342834473 }, { "label": "completed_group_2", "mean": 0.07822999358177185, "median": 0.07778562605381012, "p05": 0.06707640737295151, "p95": 0.09089364111423492 }, { "label": "completed_group_3", "mean": 0.116765096783638, "median": 0.11516137421131134, "p05": 0.10162825882434845, "p95": 0.13653048872947693 }, { "label": "completed_group_4", "mean": 0.1540035903453827, "median": 0.15287795662879944, "p05": 0.12593276798725128, "p95": 0.18590238690376282 }, { "label": "current_group_partial", "mean": 0.2901708781719208, "median": 0.29104921221733093, "p05": 0.25643545389175415, "p95": 0.3205650746822357 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.6690887212753296, "group": 5, "latest_source_mass": 0.3342471718788147, "layer": 20, "max_source_mass": 0.3342471718788147, "mixer_index": 39, "normalized_entropy": 0.9315361520005944, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.19029226899147034, "median": 0.19644704461097717, "p05": 0.09456793963909149, "p95": 0.30468082427978516 }, { "label": "completed_group_1", "mean": 0.1100241169333458, "median": 0.10808776319026947, "p05": 0.09035106748342514, "p95": 0.1359245330095291 }, { "label": "completed_group_2", "mean": 0.11471512168645859, "median": 0.11494918912649155, "p05": 0.08799737691879272, "p95": 0.14024731516838074 }, { "label": "completed_group_3", "mean": 0.09999909996986389, "median": 0.09682698547840118, "p05": 0.07921088486909866, "p95": 0.1264391839504242 }, { "label": "completed_group_4", "mean": 0.1507222205400467, "median": 0.1476232260465622, "p05": 0.11397356539964676, "p95": 0.19991227984428406 }, { "label": "current_group_partial", "mean": 0.3342471718788147, "median": 0.33802270889282227, "p05": 0.2548266053199768, "p95": 0.40130624175071716 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.7458693981170654, "group": 6, "latest_source_mass": 0.2361394464969635, "layer": 21, "max_source_mass": 0.2361394464969635, "mixer_index": 40, "normalized_entropy": 0.9743882636597644, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.20528709888458252, "median": 0.2140858769416809, "p05": 0.13168928027153015, "p95": 0.27098023891448975 }, { "label": "completed_group_1", "mean": 0.1722964197397232, "median": 0.17238952219486237, "p05": 0.15053534507751465, "p95": 0.19365043938159943 }, { "label": "completed_group_2", "mean": 0.10435444861650467, "median": 0.10361101478338242, "p05": 0.08888213336467743, "p95": 0.12212536484003067 }, { "label": "completed_group_3", "mean": 0.12075909972190857, "median": 0.1180175170302391, "p05": 0.10358668118715286, "p95": 0.14458929002285004 }, { "label": "completed_group_4", "mean": 0.16116347908973694, "median": 0.15841388702392578, "p05": 0.1337081342935562, "p95": 0.19791515171527863 }, { "label": "completed_group_5", "mean": 0.2361394464969635, "median": 0.23709020018577576, "p05": 0.20446214079856873, "p95": 0.26509594917297363 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.4612125158309937, "group": 6, "latest_source_mass": 0.5543455481529236, "layer": 21, "max_source_mass": 0.5543455481529236, "mixer_index": 41, "normalized_entropy": 0.7509146897354808, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.08934466540813446, "median": 0.07911523431539536, "p05": 0.04488275945186615, "p95": 0.1737232506275177 }, { "label": "completed_group_1", "mean": 0.054379627108573914, "median": 0.05344422906637192, "p05": 0.04221750423312187, "p95": 0.06924191862344742 }, { "label": "completed_group_2", "mean": 0.07567165046930313, "median": 0.07438549399375916, "p05": 0.05811355262994766, "p95": 0.09632256627082825 }, { "label": "completed_group_3", "mean": 0.05758609622716904, "median": 0.05612001568078995, "p05": 0.044792283326387405, "p95": 0.07347827404737473 }, { "label": "completed_group_4", "mean": 0.07884950935840607, "median": 0.07757881283760071, "p05": 0.05758974328637123, "p95": 0.10452556610107422 }, { "label": "completed_group_5", "mean": 0.08982288837432861, "median": 0.08912864327430725, "p05": 0.0663854256272316, "p95": 0.11604885756969452 }, { "label": "current_group_partial", "mean": 0.5543455481529236, "median": 0.5596089363098145, "p05": 0.45455610752105713, "p95": 0.6367484331130981 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.7241814136505127, "group": 6, "latest_source_mass": 0.39484071731567383, "layer": 22, "max_source_mass": 0.39484071731567383, "mixer_index": 42, "normalized_entropy": 0.886053970419732, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.13360010087490082, "median": 0.14072942733764648, "p05": 0.07699814438819885, "p95": 0.17506597936153412 }, { "label": "completed_group_1", "mean": 0.10609637200832367, "median": 0.10586297512054443, "p05": 0.08648421615362167, "p95": 0.12602579593658447 }, { "label": "completed_group_2", "mean": 0.06363911926746368, "median": 0.06324908882379532, "p05": 0.054980404675006866, "p95": 0.07316673547029495 }, { "label": "completed_group_3", "mean": 0.07426095008850098, "median": 0.07362888753414154, "p05": 0.06318790465593338, "p95": 0.08665015548467636 }, { "label": "completed_group_4", "mean": 0.09383076429367065, "median": 0.09294568747282028, "p05": 0.07638302445411682, "p95": 0.11445100605487823 }, { "label": "completed_group_5", "mean": 0.13373197615146637, "median": 0.13327741622924805, "p05": 0.11399447172880173, "p95": 0.15641780197620392 }, { "label": "current_group_partial", "mean": 0.39484071731567383, "median": 0.3879151940345764, "p05": 0.3302172124385834, "p95": 0.4852263331413269 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.3979300260543823, "group": 6, "latest_source_mass": 0.5830056071281433, "layer": 22, "max_source_mass": 0.5830056071281433, "mixer_index": 43, "normalized_entropy": 0.7183939231382495, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.08959861099720001, "median": 0.07544538378715515, "p05": 0.04165133833885193, "p95": 0.20099139213562012 }, { "label": "completed_group_1", "mean": 0.05414731800556183, "median": 0.05336771905422211, "p05": 0.04069708660244942, "p95": 0.07046990841627121 }, { "label": "completed_group_2", "mean": 0.07668881118297577, "median": 0.07607124000787735, "p05": 0.0555989071726799, "p95": 0.09980960935354233 }, { "label": "completed_group_3", "mean": 0.05603286623954773, "median": 0.054573021829128265, "p05": 0.04162490367889404, "p95": 0.0730118378996849 }, { "label": "completed_group_4", "mean": 0.06979228556156158, "median": 0.06884932518005371, "p05": 0.04576495662331581, "p95": 0.09806846082210541 }, { "label": "completed_group_5", "mean": 0.07073447108268738, "median": 0.07023533433675766, "p05": 0.04956750199198723, "p95": 0.09379896521568298 }, { "label": "current_group_partial", "mean": 0.5830056071281433, "median": 0.5862616896629333, "p05": 0.4658474922180176, "p95": 0.6849814057350159 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.7459560632705688, "group": 6, "latest_source_mass": 0.37691548466682434, "layer": 23, "max_source_mass": 0.37691548466682434, "mixer_index": 44, "normalized_entropy": 0.897243926765161, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.14318783581256866, "median": 0.1493408977985382, "p05": 0.08292607218027115, "p95": 0.19464610517024994 }, { "label": "completed_group_1", "mean": 0.1094653457403183, "median": 0.10858799517154694, "p05": 0.09432064741849899, "p95": 0.1279931515455246 }, { "label": "completed_group_2", "mean": 0.06349228322505951, "median": 0.06325754523277283, "p05": 0.05439673364162445, "p95": 0.0737837627530098 }, { "label": "completed_group_3", "mean": 0.07859592139720917, "median": 0.0775163471698761, "p05": 0.06603655964136124, "p95": 0.09357087314128876 }, { "label": "completed_group_4", "mean": 0.09567482024431229, "median": 0.09468592703342438, "p05": 0.07671141624450684, "p95": 0.11774884164333344 }, { "label": "completed_group_5", "mean": 0.13266831636428833, "median": 0.13199329376220703, "p05": 0.11244359612464905, "p95": 0.15504513680934906 }, { "label": "current_group_partial", "mean": 0.37691548466682434, "median": 0.37180638313293457, "p05": 0.31207239627838135, "p95": 0.4595233201980591 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.4055393934249878, "group": 6, "latest_source_mass": 0.5772827863693237, "layer": 23, "max_source_mass": 0.5772827863693237, "mixer_index": 45, "normalized_entropy": 0.7223043644164862, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.10154955089092255, "median": 0.08766824007034302, "p05": 0.04901552200317383, "p95": 0.20287738740444183 }, { "label": "completed_group_1", "mean": 0.05619276314973831, "median": 0.055477917194366455, "p05": 0.041608311235904694, "p95": 0.07371112704277039 }, { "label": "completed_group_2", "mean": 0.07791627198457718, "median": 0.07736755907535553, "p05": 0.055920060724020004, "p95": 0.10158214718103409 }, { "label": "completed_group_3", "mean": 0.053915880620479584, "median": 0.05215863138437271, "p05": 0.03798118978738785, "p95": 0.07156119495630264 }, { "label": "completed_group_4", "mean": 0.06398355215787888, "median": 0.06229078769683838, "p05": 0.040616974234580994, "p95": 0.09270931780338287 }, { "label": "completed_group_5", "mean": 0.06915920227766037, "median": 0.0684593990445137, "p05": 0.046986356377601624, "p95": 0.09408729523420334 }, { "label": "current_group_partial", "mean": 0.5772827863693237, "median": 0.5824689269065857, "p05": 0.4518115520477295, "p95": 0.6926202178001404 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.7853620052337646, "group": 6, "latest_source_mass": 0.3407441973686218, "layer": 24, "max_source_mass": 0.3407441973686218, "mixer_index": 46, "normalized_entropy": 0.9174945750195659, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.15284113585948944, "median": 0.1593906283378601, "p05": 0.08678612112998962, "p95": 0.211342453956604 }, { "label": "completed_group_1", "mean": 0.12206703424453735, "median": 0.12153112143278122, "p05": 0.1060115247964859, "p95": 0.1399766057729721 }, { "label": "completed_group_2", "mean": 0.06508444249629974, "median": 0.06482555717229843, "p05": 0.055350616574287415, "p95": 0.07557765394449234 }, { "label": "completed_group_3", "mean": 0.08347313106060028, "median": 0.08190013468265533, "p05": 0.06966532766819, "p95": 0.10050469636917114 }, { "label": "completed_group_4", "mean": 0.09811970591545105, "median": 0.09664031863212585, "p05": 0.07688035070896149, "p95": 0.12369741499423981 }, { "label": "completed_group_5", "mean": 0.13767032325267792, "median": 0.1373325139284134, "p05": 0.11642646044492722, "p95": 0.1606694608926773 }, { "label": "current_group_partial", "mean": 0.3407441973686218, "median": 0.33805108070373535, "p05": 0.271301805973053, "p95": 0.41781002283096313 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.509842872619629, "group": 6, "latest_source_mass": 0.5227814316749573, "layer": 24, "max_source_mass": 0.5227814316749573, "mixer_index": 47, "normalized_entropy": 0.7759057494780099, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.11788781732320786, "median": 0.10097194463014603, "p05": 0.06265684217214584, "p95": 0.2408325970172882 }, { "label": "completed_group_1", "mean": 0.06822571158409119, "median": 0.0673636943101883, "p05": 0.05209728330373764, "p95": 0.08648915588855743 }, { "label": "completed_group_2", "mean": 0.08556114137172699, "median": 0.08463940769433975, "p05": 0.06234566867351532, "p95": 0.1125786080956459 }, { "label": "completed_group_3", "mean": 0.06067090481519699, "median": 0.058974385261535645, "p05": 0.043267179280519485, "p95": 0.07997901737689972 }, { "label": "completed_group_4", "mean": 0.07228855788707733, "median": 0.06972275674343109, "p05": 0.04689513146877289, "p95": 0.106512151658535 }, { "label": "completed_group_5", "mean": 0.07258446514606476, "median": 0.07082539796829224, "p05": 0.050949547439813614, "p95": 0.09970614314079285 }, { "label": "current_group_partial", "mean": 0.5227814316749573, "median": 0.5290273427963257, "p05": 0.38773614168167114, "p95": 0.6392589807510376 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.839367389678955, "group": 7, "latest_source_mass": 0.29583215713500977, "layer": 25, "max_source_mass": 0.29583215713500977, "mixer_index": 48, "normalized_entropy": 0.9452478525649903, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.15906387567520142, "median": 0.15846745669841766, "p05": 0.09519976377487183, "p95": 0.24517810344696045 }, { "label": "completed_group_1", "mean": 0.13139380514621735, "median": 0.1308426856994629, "p05": 0.11478274315595627, "p95": 0.15000829100608826 }, { "label": "completed_group_2", "mean": 0.07717745751142502, "median": 0.07715500891208649, "p05": 0.06616044044494629, "p95": 0.08791667222976685 }, { "label": "completed_group_3", "mean": 0.09607303142547607, "median": 0.09436552226543427, "p05": 0.08152313530445099, "p95": 0.11384639143943787 }, { "label": "completed_group_4", "mean": 0.1107850894331932, "median": 0.1089654266834259, "p05": 0.08814431726932526, "p95": 0.139452263712883 }, { "label": "completed_group_5", "mean": 0.12967455387115479, "median": 0.1299237608909607, "p05": 0.10832332074642181, "p95": 0.15138982236385345 }, { "label": "completed_group_6", "mean": 0.29583215713500977, "median": 0.29647016525268555, "p05": 0.2276897132396698, "p95": 0.3604946434497833 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.5841834545135498, "group": 7, "latest_source_mass": 0.40767741203308105, "layer": 25, "max_source_mass": 0.40767741203308105, "mixer_index": 49, "normalized_entropy": 0.7618312045616817, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.07896879315376282, "median": 0.06809917092323303, "p05": 0.025932887569069862, "p95": 0.19368338584899902 }, { "label": "completed_group_1", "mean": 0.048752427101135254, "median": 0.04767637699842453, "p05": 0.034443892538547516, "p95": 0.0667153000831604 }, { "label": "completed_group_2", "mean": 0.055707164108753204, "median": 0.0549345538020134, "p05": 0.038696639239788055, "p95": 0.0750277042388916 }, { "label": "completed_group_3", "mean": 0.04139826446771622, "median": 0.03878612071275711, "p05": 0.026673054322600365, "p95": 0.05746251344680786 }, { "label": "completed_group_4", "mean": 0.04449018836021423, "median": 0.04196460545063019, "p05": 0.026162121444940567, "p95": 0.07164917886257172 }, { "label": "completed_group_5", "mean": 0.04285799711942673, "median": 0.04109475016593933, "p05": 0.029393533244729042, "p95": 0.06277710199356079 }, { "label": "completed_group_6", "mean": 0.2801477909088135, "median": 0.2763082981109619, "p05": 0.1455708146095276, "p95": 0.4254714846611023 }, { "label": "current_group_partial", "mean": 0.40767741203308105, "median": 0.4120897948741913, "p05": 0.21399420499801636, "p95": 0.5853031277656555 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.8125314712524414, "group": 7, "latest_source_mass": 0.3700707256793976, "layer": 26, "max_source_mass": 0.3700707256793976, "mixer_index": 50, "normalized_entropy": 0.8716433883436914, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.10889513790607452, "median": 0.11333495378494263, "p05": 0.04850132763385773, "p95": 0.1620434820652008 }, { "label": "completed_group_1", "mean": 0.08656036853790283, "median": 0.08640051633119583, "p05": 0.0650845319032669, "p95": 0.1084962785243988 }, { "label": "completed_group_2", "mean": 0.048123255372047424, "median": 0.04800213500857353, "p05": 0.03576180338859558, "p95": 0.06107594817876816 }, { "label": "completed_group_3", "mean": 0.06561170518398285, "median": 0.06379580497741699, "p05": 0.050968848168849945, "p95": 0.08332888036966324 }, { "label": "completed_group_4", "mean": 0.06917259097099304, "median": 0.06724178791046143, "p05": 0.050207968801259995, "p95": 0.09373342245817184 }, { "label": "completed_group_5", "mean": 0.08286689221858978, "median": 0.08183546364307404, "p05": 0.06535022705793381, "p95": 0.10340933501720428 }, { "label": "completed_group_6", "mean": 0.16869932413101196, "median": 0.16509854793548584, "p05": 0.11059704422950745, "p95": 0.23654375970363617 }, { "label": "current_group_partial", "mean": 0.3700707256793976, "median": 0.3694133758544922, "p05": 0.23441238701343536, "p95": 0.5025027990341187 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.50089693069458, "group": 7, "latest_source_mass": 0.49096518754959106, "layer": 26, "max_source_mass": 0.49096518754959106, "mixer_index": 51, "normalized_entropy": 0.7217788529328457, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.07974998652935028, "median": 0.07402504235506058, "p05": 0.04579866677522659, "p95": 0.1359037309885025 }, { "label": "completed_group_1", "mean": 0.04166398197412491, "median": 0.04038450866937637, "p05": 0.027992289513349533, "p95": 0.05969971418380737 }, { "label": "completed_group_2", "mean": 0.04607272148132324, "median": 0.044438350945711136, "p05": 0.030038591474294662, "p95": 0.06808309257030487 }, { "label": "completed_group_3", "mean": 0.03586764633655548, "median": 0.03317485377192497, "p05": 0.02203454077243805, "p95": 0.05572832003235817 }, { "label": "completed_group_4", "mean": 0.04223340004682541, "median": 0.03930673003196716, "p05": 0.02465122565627098, "p95": 0.06968094408512115 }, { "label": "completed_group_5", "mean": 0.04095568507909775, "median": 0.03919431194663048, "p05": 0.026239857077598572, "p95": 0.06058088690042496 }, { "label": "completed_group_6", "mean": 0.22249138355255127, "median": 0.2125401496887207, "p05": 0.10647299885749817, "p95": 0.3711881935596466 }, { "label": "current_group_partial", "mean": 0.49096518754959106, "median": 0.4988659620285034, "p05": 0.2878011167049408, "p95": 0.6676807403564453 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.8059929609298706, "group": 7, "latest_source_mass": 0.37139976024627686, "layer": 27, "max_source_mass": 0.37139976024627686, "mixer_index": 52, "normalized_entropy": 0.8684990295379666, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.10711923241615295, "median": 0.10972198843955994, "p05": 0.04896187782287598, "p95": 0.16201043128967285 }, { "label": "completed_group_1", "mean": 0.07651707530021667, "median": 0.07666032761335373, "p05": 0.05141128972172737, "p95": 0.10152198374271393 }, { "label": "completed_group_2", "mean": 0.058284737169742584, "median": 0.05812416970729828, "p05": 0.039273619651794434, "p95": 0.07745087146759033 }, { "label": "completed_group_3", "mean": 0.06609587371349335, "median": 0.06383530795574188, "p05": 0.04481930285692215, "p95": 0.09336265176534653 }, { "label": "completed_group_4", "mean": 0.07139429450035095, "median": 0.0694514587521553, "p05": 0.0485839769244194, "p95": 0.10034126043319702 }, { "label": "completed_group_5", "mean": 0.07493151724338531, "median": 0.07319007813930511, "p05": 0.051876723766326904, "p95": 0.10402432829141617 }, { "label": "completed_group_6", "mean": 0.17425751686096191, "median": 0.16937679052352905, "p05": 0.10449051856994629, "p95": 0.2624811828136444 }, { "label": "current_group_partial", "mean": 0.37139976024627686, "median": 0.3732905089855194, "p05": 0.23830299079418182, "p95": 0.5014542937278748 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.5622732639312744, "group": 7, "latest_source_mass": 0.4768889844417572, "layer": 27, "max_source_mass": 0.4768889844417572, "mixer_index": 53, "normalized_entropy": 0.7512946301290215, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.08775827288627625, "median": 0.08504067361354828, "p05": 0.0428270623087883, "p95": 0.144285649061203 }, { "label": "completed_group_1", "mean": 0.047765426337718964, "median": 0.04646281152963638, "p05": 0.031814731657505035, "p95": 0.06807165592908859 }, { "label": "completed_group_2", "mean": 0.053928617388010025, "median": 0.052262187004089355, "p05": 0.035437777638435364, "p95": 0.0784272700548172 }, { "label": "completed_group_3", "mean": 0.04293380305171013, "median": 0.03996016085147858, "p05": 0.02556057646870613, "p95": 0.06623870134353638 }, { "label": "completed_group_4", "mean": 0.04174989461898804, "median": 0.03903091698884964, "p05": 0.025013400241732597, "p95": 0.06869364529848099 }, { "label": "completed_group_5", "mean": 0.04734005779027939, "median": 0.04609068110585213, "p05": 0.029884062707424164, "p95": 0.06911227852106094 }, { "label": "completed_group_6", "mean": 0.20163494348526, "median": 0.19222964346408844, "p05": 0.09963390231132507, "p95": 0.3307305574417114 }, { "label": "current_group_partial", "mean": 0.4768889844417572, "median": 0.48383551836013794, "p05": 0.276212215423584, "p95": 0.6487575173377991 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.8757455348968506, "group": 7, "latest_source_mass": 0.3109493851661682, "layer": 28, "max_source_mass": 0.3109493851661682, "mixer_index": 54, "normalized_entropy": 0.9020429270551009, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.11563669890165329, "median": 0.12087491154670715, "p05": 0.05733206868171692, "p95": 0.1688953936100006 }, { "label": "completed_group_1", "mean": 0.08108697831630707, "median": 0.08185897022485733, "p05": 0.05175650864839554, "p95": 0.10753051191568375 }, { "label": "completed_group_2", "mean": 0.07280858606100082, "median": 0.07357072830200195, "p05": 0.04870978742837906, "p95": 0.0948689877986908 }, { "label": "completed_group_3", "mean": 0.07553227245807648, "median": 0.07362619787454605, "p05": 0.056317005306482315, "p95": 0.09756095707416534 }, { "label": "completed_group_4", "mean": 0.07606014609336853, "median": 0.07456053793430328, "p05": 0.05528895929455757, "p95": 0.1022825762629509 }, { "label": "completed_group_5", "mean": 0.06935927271842957, "median": 0.06843917071819305, "p05": 0.04978285729885101, "p95": 0.09167985618114471 }, { "label": "completed_group_6", "mean": 0.19856664538383484, "median": 0.19508783519268036, "p05": 0.1281973123550415, "p95": 0.27796900272369385 }, { "label": "current_group_partial", "mean": 0.3109493851661682, "median": 0.3105033040046692, "p05": 0.18172834813594818, "p95": 0.43916669487953186 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.6924852132797241, "group": 7, "latest_source_mass": 0.3315581679344177, "layer": 28, "max_source_mass": 0.3315581679344177, "mixer_index": 55, "normalized_entropy": 0.8139133413255193, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.08638577908277512, "median": 0.058608733117580414, "p05": 0.026544082909822464, "p95": 0.2740764319896698 }, { "label": "completed_group_1", "mean": 0.059912845492362976, "median": 0.05901291221380234, "p05": 0.03629481792449951, "p95": 0.08707985281944275 }, { "label": "completed_group_2", "mean": 0.06609142571687698, "median": 0.0647939145565033, "p05": 0.038691695779561996, "p95": 0.09703366458415985 }, { "label": "completed_group_3", "mean": 0.056745611131191254, "median": 0.05418796092271805, "p05": 0.032174739986658096, "p95": 0.08370660990476608 }, { "label": "completed_group_4", "mean": 0.06277773529291153, "median": 0.0600711889564991, "p05": 0.03501531109213829, "p95": 0.10094858705997467 }, { "label": "completed_group_5", "mean": 0.05352145433425903, "median": 0.05179818347096443, "p05": 0.032271772623062134, "p95": 0.08084158599376678 }, { "label": "completed_group_6", "mean": 0.2830069661140442, "median": 0.2759752869606018, "p05": 0.12981083989143372, "p95": 0.46204638481140137 }, { "label": "current_group_partial", "mean": 0.3315581679344177, "median": 0.3204721510410309, "p05": 0.13446375727653503, "p95": 0.5667697787284851 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.8980522155761719, "group": 8, "latest_source_mass": 0.20123262703418732, "layer": 29, "max_source_mass": 0.26602888107299805, "mixer_index": 56, "normalized_entropy": 0.9127701729200177, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.11603894829750061, "median": 0.09991352260112762, "p05": 0.030035395175218582, "p95": 0.2941743731498718 }, { "label": "completed_group_1", "mean": 0.08804173022508621, "median": 0.08880250155925751, "p05": 0.05664174258708954, "p95": 0.11779069900512695 }, { "label": "completed_group_2", "mean": 0.07975024729967117, "median": 0.07984748482704163, "p05": 0.05428915470838547, "p95": 0.10499341040849686 }, { "label": "completed_group_3", "mean": 0.09027000516653061, "median": 0.08898856490850449, "p05": 0.06189651042222977, "p95": 0.11996553838253021 }, { "label": "completed_group_4", "mean": 0.08430293202400208, "median": 0.0819794088602066, "p05": 0.05464346706867218, "p95": 0.12159879505634308 }, { "label": "completed_group_5", "mean": 0.07433462142944336, "median": 0.07287181913852692, "p05": 0.052520088851451874, "p95": 0.1010315865278244 }, { "label": "completed_group_6", "mean": 0.26602888107299805, "median": 0.2639757990837097, "p05": 0.15474572777748108, "p95": 0.3846370279788971 }, { "label": "completed_group_7", "mean": 0.20123262703418732, "median": 0.18140269815921783, "p05": 0.08419936150312424, "p95": 0.3782624900341034 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.7512003183364868, "group": 8, "latest_source_mass": 0.33482813835144043, "layer": 29, "max_source_mass": 0.33482813835144043, "mixer_index": 57, "normalized_entropy": 0.7970056117156376, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.07744461297988892, "median": 0.06970229744911194, "p05": 0.014401955530047417, "p95": 0.17144526541233063 }, { "label": "completed_group_1", "mean": 0.046086348593235016, "median": 0.04394831880927086, "p05": 0.028066720813512802, "p95": 0.0718703418970108 }, { "label": "completed_group_2", "mean": 0.05025433003902435, "median": 0.04861236736178398, "p05": 0.029644077643752098, "p95": 0.07761462777853012 }, { "label": "completed_group_3", "mean": 0.03748488053679466, "median": 0.03434944897890091, "p05": 0.020774798467755318, "p95": 0.05938909202814102 }, { "label": "completed_group_4", "mean": 0.04005061462521553, "median": 0.037498839199543, "p05": 0.019541850313544273, "p95": 0.06922487914562225 }, { "label": "completed_group_5", "mean": 0.041623204946517944, "median": 0.039552345871925354, "p05": 0.022081367671489716, "p95": 0.06821337342262268 }, { "label": "completed_group_6", "mean": 0.198423832654953, "median": 0.18318486213684082, "p05": 0.07548617571592331, "p95": 0.3721975088119507 }, { "label": "completed_group_7", "mean": 0.17380405962467194, "median": 0.1514919549226761, "p05": 0.05777408182621002, "p95": 0.36326974630355835 }, { "label": "current_group_partial", "mean": 0.33482813835144043, "median": 0.33602219820022583, "p05": 0.0795319601893425, "p95": 0.5979493856430054 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.0160694122314453, "group": 8, "latest_source_mass": 0.21711516380310059, "layer": 30, "max_source_mass": 0.21711516380310059, "mixer_index": 58, "normalized_entropy": 0.9175527313077866, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.09385271370410919, "median": 0.09211117774248123, "p05": 0.032377734780311584, "p95": 0.16496515274047852 }, { "label": "completed_group_1", "mean": 0.06208505108952522, "median": 0.06140740588307381, "p05": 0.0407925546169281, "p95": 0.0863042101264 }, { "label": "completed_group_2", "mean": 0.06560768187046051, "median": 0.06516212224960327, "p05": 0.041897669434547424, "p95": 0.09085705131292343 }, { "label": "completed_group_3", "mean": 0.07029052078723907, "median": 0.06780634075403214, "p05": 0.04846132919192314, "p95": 0.09917552024126053 }, { "label": "completed_group_4", "mean": 0.06935097277164459, "median": 0.06708167493343353, "p05": 0.04668639227747917, "p95": 0.09815650433301926 }, { "label": "completed_group_5", "mean": 0.056832700967788696, "median": 0.05595892667770386, "p05": 0.038822539150714874, "p95": 0.0771557092666626 }, { "label": "completed_group_6", "mean": 0.19425815343856812, "median": 0.19145764410495758, "p05": 0.12249615788459778, "p95": 0.27496030926704407 }, { "label": "completed_group_7", "mean": 0.17060703039169312, "median": 0.16317546367645264, "p05": 0.09239593893289566, "p95": 0.2726203501224518 }, { "label": "current_group_partial", "mean": 0.21711516380310059, "median": 0.21198657155036926, "p05": 0.11019522696733475, "p95": 0.3391069769859314 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 1.7837426662445068, "group": 8, "latest_source_mass": 0.29026859998703003, "layer": 30, "max_source_mass": 0.29026859998703003, "mixer_index": 59, "normalized_entropy": 0.8118162725118464, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.08836327493190765, "median": 0.07904411852359772, "p05": 0.02631576545536518, "p95": 0.17534467577934265 }, { "label": "completed_group_1", "mean": 0.04953558370471001, "median": 0.04658038914203644, "p05": 0.025181813165545464, "p95": 0.08318150043487549 }, { "label": "completed_group_2", "mean": 0.04813014343380928, "median": 0.045740917325019836, "p05": 0.026714935898780823, "p95": 0.07843323051929474 }, { "label": "completed_group_3", "mean": 0.0373263955116272, "median": 0.03401809185743332, "p05": 0.01990434154868126, "p95": 0.06290259957313538 }, { "label": "completed_group_4", "mean": 0.037015400826931, "median": 0.03405017405748367, "p05": 0.018646355718374252, "p95": 0.06544196605682373 }, { "label": "completed_group_5", "mean": 0.03390160948038101, "median": 0.03198609501123428, "p05": 0.018189728260040283, "p95": 0.05570102855563164 }, { "label": "completed_group_6", "mean": 0.1840541511774063, "median": 0.17327937483787537, "p05": 0.07966600358486176, "p95": 0.32982927560806274 }, { "label": "completed_group_7", "mean": 0.23140482604503632, "median": 0.22406929731369019, "p05": 0.08831030130386353, "p95": 0.40350067615509033 }, { "label": "current_group_partial", "mean": 0.29026859998703003, "median": 0.2745037376880646, "p05": 0.10280418395996094, "p95": 0.5399085879325867 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 1.9973840713500977, "group": 8, "latest_source_mass": 0.19101998209953308, "layer": 31, "max_source_mass": 0.20831164717674255, "mixer_index": 60, "normalized_entropy": 0.9090486661912383, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.10261696577072144, "median": 0.10389091819524765, "p05": 0.01371697522699833, "p95": 0.21163305640220642 }, { "label": "completed_group_1", "mean": 0.06039813905954361, "median": 0.06085003539919853, "p05": 0.026107093319296837, "p95": 0.09348798543214798 }, { "label": "completed_group_2", "mean": 0.06671924144029617, "median": 0.06656201928853989, "p05": 0.034684643149375916, "p95": 0.09819839894771576 }, { "label": "completed_group_3", "mean": 0.08377155661582947, "median": 0.0804532840847969, "p05": 0.05448959022760391, "p95": 0.12039553374052048 }, { "label": "completed_group_4", "mean": 0.07002593576908112, "median": 0.06737451255321503, "p05": 0.044841229915618896, "p95": 0.10655929893255234 }, { "label": "completed_group_5", "mean": 0.05768897384405136, "median": 0.056077782064676285, "p05": 0.039495617151260376, "p95": 0.08242568373680115 }, { "label": "completed_group_6", "mean": 0.20831164717674255, "median": 0.2021285891532898, "p05": 0.11652229726314545, "p95": 0.31706833839416504 }, { "label": "completed_group_7", "mean": 0.1594475358724594, "median": 0.1546279788017273, "p05": 0.06593266129493713, "p95": 0.2735573351383209 }, { "label": "current_group_partial", "mean": 0.19101998209953308, "median": 0.1713225245475769, "p05": 0.06408411264419556, "p95": 0.3850628137588501 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 1.969987154006958, "group": 8, "latest_source_mass": 0.16864418983459473, "layer": 31, "max_source_mass": 0.24503237009048462, "mixer_index": 61, "normalized_entropy": 0.8965797917640489, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.08120635151863098, "median": 0.061164144426584244, "p05": 0.0259944386780262, "p95": 0.23622018098831177 }, { "label": "completed_group_1", "mean": 0.07894632965326309, "median": 0.07658770680427551, "p05": 0.0496862456202507, "p95": 0.11596276611089706 }, { "label": "completed_group_2", "mean": 0.08615730702877045, "median": 0.08467075228691101, "p05": 0.050409119576215744, "p95": 0.12811316549777985 }, { "label": "completed_group_3", "mean": 0.07609779387712479, "median": 0.0722871869802475, "p05": 0.04099677503108978, "p95": 0.11950768530368805 }, { "label": "completed_group_4", "mean": 0.058728985488414764, "median": 0.056080836802721024, "p05": 0.03323862701654434, "p95": 0.09344407916069031 }, { "label": "completed_group_5", "mean": 0.05076971277594566, "median": 0.04946017265319824, "p05": 0.03047419898211956, "p95": 0.07563400268554688 }, { "label": "completed_group_6", "mean": 0.24503237009048462, "median": 0.24110376834869385, "p05": 0.10613486170768738, "p95": 0.3991503119468689 }, { "label": "completed_group_7", "mean": 0.1544169783592224, "median": 0.1400495171546936, "p05": 0.06199025735259056, "p95": 0.2995939254760742 }, { "label": "current_group_partial", "mean": 0.16864418983459473, "median": 0.15222924947738647, "p05": 0.0543956533074379, "p95": 0.3329208791255951 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.071648359298706, "group": 8, "latest_source_mass": 0.11520501971244812, "layer": 32, "max_source_mass": 0.20955611765384674, "mixer_index": 62, "normalized_entropy": 0.9428478002054053, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.10569609701633453, "median": 0.08689188957214355, "p05": 0.03173588588833809, "p95": 0.24571751058101654 }, { "label": "completed_group_1", "mean": 0.09922641515731812, "median": 0.09795533865690231, "p05": 0.06407676637172699, "p95": 0.14120742678642273 }, { "label": "completed_group_2", "mean": 0.08180510997772217, "median": 0.080217644572258, "p05": 0.054607540369033813, "p95": 0.11442367732524872 }, { "label": "completed_group_3", "mean": 0.10646799206733704, "median": 0.10510502755641937, "p05": 0.07415887713432312, "p95": 0.14216329157352448 }, { "label": "completed_group_4", "mean": 0.0826798751950264, "median": 0.08022409677505493, "p05": 0.05491747707128525, "p95": 0.11860857903957367 }, { "label": "completed_group_5", "mean": 0.06615597754716873, "median": 0.06485182791948318, "p05": 0.04477996751666069, "p95": 0.09238749742507935 }, { "label": "completed_group_6", "mean": 0.20955611765384674, "median": 0.206287682056427, "p05": 0.129981130361557, "p95": 0.2996031939983368 }, { "label": "completed_group_7", "mean": 0.13320739567279816, "median": 0.12465103715658188, "p05": 0.06303662061691284, "p95": 0.23065781593322754 }, { "label": "current_group_partial", "mean": 0.11520501971244812, "median": 0.10386130958795547, "p05": 0.03912060335278511, "p95": 0.22735844552516937 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 1.9781701564788818, "group": 8, "latest_source_mass": 0.16112206876277924, "layer": 32, "max_source_mass": 0.22251944243907928, "mixer_index": 63, "normalized_entropy": 0.9003040366848136, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.09446848928928375, "median": 0.074150450527668, "p05": 0.042592622339725494, "p95": 0.19443637132644653 }, { "label": "completed_group_1", "mean": 0.07056106626987457, "median": 0.06868506222963333, "p05": 0.04818308353424072, "p95": 0.0982733890414238 }, { "label": "completed_group_2", "mean": 0.08033448457717896, "median": 0.08047192543745041, "p05": 0.048296794295310974, "p95": 0.11253741383552551 }, { "label": "completed_group_3", "mean": 0.0658978745341301, "median": 0.06256154924631119, "p05": 0.04223886504769325, "p95": 0.0967884361743927 }, { "label": "completed_group_4", "mean": 0.058234501630067825, "median": 0.0563921295106411, "p05": 0.03529864549636841, "p95": 0.08846980333328247 }, { "label": "completed_group_5", "mean": 0.04316519945859909, "median": 0.042268507182598114, "p05": 0.029443470761179924, "p95": 0.05996086448431015 }, { "label": "completed_group_6", "mean": 0.2036968618631363, "median": 0.19645483791828156, "p05": 0.09398645162582397, "p95": 0.3395216464996338 }, { "label": "completed_group_7", "mean": 0.22251944243907928, "median": 0.2130982130765915, "p05": 0.11867167800664902, "p95": 0.35465875267982483 }, { "label": "current_group_partial", "mean": 0.16112206876277924, "median": 0.14023703336715698, "p05": 0.06008770316839218, "p95": 0.3247341811656952 } ], "sources": 9 }, { "branch": "output", "entropy_mean": 1.6036605834960938, "group": null, "latest_source_mass": 0.31639498472213745, "layer": null, "max_source_mass": 0.3412485122680664, "mixer_index": null, "normalized_entropy": 0.7298573846467136, "offset": null, "source_summaries": [ { "label": "embedding", "mean": 0.09759747982025146, "median": 0.09915056824684143, "p05": 0.025461740791797638, "p95": 0.16398483514785767 }, { "label": "completed_group_1", "mean": 0.04215940088033676, "median": 0.04050712287425995, "p05": 0.025720715522766113, "p95": 0.06398092210292816 }, { "label": "completed_group_2", "mean": 0.041145481169223785, "median": 0.03881259262561798, "p05": 0.023510465398430824, "p95": 0.0662468671798706 }, { "label": "completed_group_3", "mean": 0.029554028064012527, "median": 0.027420252561569214, "p05": 0.01627083122730255, "p95": 0.04924973472952843 }, { "label": "completed_group_4", "mean": 0.0289996936917305, "median": 0.027226522564888, "p05": 0.01560397818684578, "p95": 0.048351090401411057 }, { "label": "completed_group_5", "mean": 0.03647579252719879, "median": 0.0347352996468544, "p05": 0.020744025707244873, "p95": 0.058257561177015305 }, { "label": "completed_group_6", "mean": 0.06642461568117142, "median": 0.057809896767139435, "p05": 0.022837381809949875, "p95": 0.14078694581985474 }, { "label": "completed_group_7", "mean": 0.3412485122680664, "median": 0.3336990475654602, "p05": 0.0819513276219368, "p95": 0.6423859596252441 }, { "label": "completed_group_8", "mean": 0.31639498472213745, "median": 0.30713334679603577, "p05": 0.05841828137636185, "p95": 0.6156357526779175 } ], "sources": 9 } ], "mode": "uniform_except_groups_6_7", "optimizer_state_after": "ed03e6fbd4a12d8b063dcb22e0437754285f54d585374cd52fbd534f05d24637", "optimizer_state_before": "ed03e6fbd4a12d8b063dcb22e0437754285f54d585374cd52fbd534f05d24637", "optimizer_state_unchanged": true, "positions": { "attention_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "5e6ecbcd78543fc42d4098a51169dca446d404ce384015643e84d2c78433c949", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.0933690071105957, 0.24537226557731628, 0.07698935270309448, 0.0523640513420105, 0.09380833059549332, 0.03647654876112938, 0.039120711386203766, 0.06371714919805527, 0.05156813934445381, 0.03728010877966881, 0.03983496502041817, 0.040284499526023865, 0.046871740370988846, 0.04314620420336723, 0.04033593460917473, 0.04471700266003609, 0.08397205173969269, 0.0528092160820961, 0.04443390294909477, 0.07560808956623077, 0.03708760067820549, 0.03599191829562187, 0.033304885029792786, 0.04061548039317131, 0.025487789884209633, 0.021565141156315804, 0.02342291921377182, 0.017244853079319, 0.024166326969861984, 0.016819356009364128, 0.021590016782283783, 0.026201345026493073 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 8.033875474211527e-06, "first_to_last_ratio": 1.4258966620890539, "imbalance_abs_log_ratio": 0.3548008523880228, "last_quartile_mean": 5.634262066678275e-06, "mean": 6.824666463955964e-06, "normalized": [ 1.3817175053293855, 1.4804492128492015, 1.2768822237285216, 1.1433064378254503, 1.1306525826694747, 1.0654997945691926, 1.0111021570021848, 0.9278472076508293, 0.8519589257241937, 0.7842307758528145, 0.7445131668113949, 0.7100583578580685, 0.7217497673368972, 0.6945877965497559, 0.6750241757973762, 0.6552200102586146, 0.6913561672991363, 0.6794511327274709, 0.6949220938332125, 0.718906874452968, 2.4003760336362987, 2.0411608423609806, 1.624335374645907, 1.290105390423236, 1.595179081017655, 1.3978180141198187, 1.0513376369798262, 0.6710162732056626, 0.5205881261306791, 0.48846107107415765, 0.4495472880799729, 0.43063850219966204 ], "peak_layer": 21, "peak_normalized": 2.4003760336362987, "population_cv": 0.45909521114658286, "rest_mean": 5.82595065747295e-06, "spike_contrast": 2.097122433279202, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.2217731818964239e-05, "top_five_layers": [ 21, 22, 23, 25, 2 ], "top_five_spike_overlap": 4 }, "values": [ 9.429761121282354e-06, 1.0103572094521951e-05, 8.714295290701557e-06, 7.802685104252305e-06, 7.716326763329562e-06, 7.271680715348339e-06, 6.900434982526349e-06, 6.3322477217298e-06, 5.814335509057855e-06, 5.35211347596487e-06, 5.081054041511379e-06, 4.845911462325603e-06, 4.925701432512142e-06, 4.740330041386187e-06, 4.6068148549238686e-06, 4.47165803052485e-06, 4.7182752496155445e-06, 4.6370273594220635e-06, 4.7426115088455845e-06, 4.906299636786571e-06, 1.638176581764128e-05, 1.393024194840109e-05, 1.1085547157563269e-05, 8.804538992990274e-06, 1.0886565178225283e-05, 9.53964172367705e-06, 7.17502871339093e-06, 4.579462256515399e-06, 3.552840325937723e-06, 3.333583890707814e-06, 3.068010300921742e-06, 2.93896414405026e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 3.183153444297204e-05, "first_to_last_ratio": 1.4131276884719195, "imbalance_abs_log_ratio": 0.3458054665494247, "last_quartile_mean": 2.25255896566523e-05, "mean": 2.7195942266189377e-05, "normalized": [ 1.3914477134872245, 1.4843172319639206, 1.2741857869430397, 1.1347168393714524, 1.113836808577119, 1.0480554815017753, 0.997051836032532, 0.9200011915489417, 0.8457179637933491, 0.7806635384243056, 0.7404860999654215, 0.7096167890148819, 0.7229480245131014, 0.6960855685974651, 0.6774994785301266, 0.6578219657499239, 0.6991460735916578, 0.6833992343002161, 0.6966813096259906, 0.7181039082011031, 2.404683680249898, 2.0538004412526782, 1.6357067521806194, 1.2878643541086146, 1.5982357317051483, 1.3983424443937627, 1.0506111416230242, 0.6723101708576006, 0.5245716127811398, 0.4928659769377425, 0.4541380612778038, 0.4350867888984191 ], "peak_layer": 21, "peak_normalized": 2.404683680249898, "population_cv": 0.46017131642553544, "rest_mean": 2.3186765853703644e-05, "spike_contrast": 2.1066109522044534, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.884549489361234e-05, "top_five_layers": [ 21, 22, 23, 25, 2 ], "top_five_spike_overlap": 4 }, "values": [ 3.784173168241978e-05, 4.036740574520081e-05, 3.465268309810199e-05, 3.0859693652018905e-05, 3.029184154001996e-05, 2.850285636668559e-05, 2.7115764169138856e-05, 2.502029929019045e-05, 2.300009691680316e-05, 2.1230880520306528e-05, 2.013821722357534e-05, 1.9298697225167416e-05, 1.9661252736113966e-05, 1.8930702935904264e-05, 1.842523670347873e-05, 1.7890088201966137e-05, 1.9013936253031716e-05, 1.8585686120786704e-05, 1.8946904674521647e-05, 1.9529512428562157e-05, 6.539763853652403e-05, 5.585503822658211e-05, 4.448458639672026e-05, 3.502468462102115e-05, 4.346552668721415e-05, 3.80292403860949e-05, 2.857235995179508e-05, 1.8284108591615222e-05, 1.4266219295677729e-05, 1.3403954653767869e-05, 1.2350712495390326e-05, 1.1832595191663131e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.028228522976860404, "first_to_last_ratio": 1.413127712661979, "imbalance_abs_log_ratio": 0.3458054836675234, "last_quartile_mean": 0.019975917763076723, "mean": 0.024117633613059297, "normalized": [ 1.3914478022974348, 1.4843171878139365, 1.2741857202137106, 1.1347169067842968, 1.1138367838547423, 1.048055480230206, 0.9970518390356672, 0.9200011216452147, 0.845717923960402, 0.7806635248251631, 0.7404861420672175, 0.7096167975534927, 0.7229479880136591, 0.6960855770649286, 0.6774994664649109, 0.6578219951192215, 0.6991461137310863, 0.6833992710592495, 0.6966812649448638, 0.7181039403645707, 2.404683769727251, 2.0538005161757025, 1.635706709692463, 1.2878643759625878, 1.5982356019248984, 1.39834242804845, 1.050611153461755, 0.6723101932455761, 0.5245716055436657, 0.4928659578783944, 0.45413805912545646, 0.43508678216982477 ], "peak_layer": 21, "peak_normalized": 2.404683769727251, "population_cv": 0.4601713229358668, "rest_mean": 0.02056225585854716, "spike_contrast": 2.106610956765199, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.04331667348742485, "top_five_layers": [ 21, 22, 23, 25, 2 ], "top_five_spike_overlap": 4 }, "values": [ 0.033558428287506104, 0.035798218101263046, 0.030730344355106354, 0.02736668661236763, 0.026863107457756996, 0.02527661807835102, 0.024046530947089195, 0.022188249975442886, 0.02039671503007412, 0.018827756866812706, 0.017858773469924927, 0.01711427792906761, 0.017435794696211815, 0.0167879369109869, 0.016339683905243874, 0.015865109860897064, 0.016861749812960625, 0.016481973230838776, 0.01680230349302292, 0.0173189677298069, 0.057995282113552094, 0.04953280836343765, 0.03944937512278557, 0.031060241162776947, 0.03854566067457199, 0.03372471034526825, 0.025338254868984222, 0.016214530915021896, 0.012651425786316395, 0.011886760592460632, 0.01095273531973362, 0.010493263602256775 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 1.2542293106321267e-06, "first_to_last_ratio": 0.8293947691896986, "imbalance_abs_log_ratio": 0.18705903792991904, "last_quartile_mean": 1.5122223544494773e-06, "mean": 1.4018511329538796e-06, "normalized": [ 1.7194118549791988, 0.9886355450281227, 0.9006921139868579, 0.8067392657086441, 0.7084339874539142, 0.684281970814399, 0.6715049949248708, 0.6778609018872602, 0.6320283601536156, 0.6003502379576225, 0.5871671655671831, 0.5890108801752642, 0.5949621909433359, 0.6012310801233034, 0.611059913007814, 0.6229822007249801, 0.6477819848734375, 0.6740515423487576, 0.7000514906743464, 0.7366179361425164, 2.4969584793824993, 2.370243071082993, 2.0543393257361138, 1.693743636606077, 2.059375327835681, 1.8488175968426048, 1.434697294074145, 0.8812388143051648, 0.6604582780258951, 0.6279603396486315, 0.5764254569437794, 0.540886762040971 ], "peak_layer": 21, "peak_normalized": 2.4969584793824993, "population_cv": 0.5863807965140019, "rest_mean": 1.1072204541859619e-06, "spike_contrast": 2.703036045790005, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.992856798300636e-06, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 2.4103594569169218e-06, 1.38591985887615e-06, 1.2626362604351016e-06, 1.1309283536320436e-06, 9.931189879353042e-07, 9.592614560460788e-07, 9.413500379196194e-07, 9.502600732957944e-07, 8.860096727403288e-07, 8.416016612500243e-07, 8.231209562836739e-07, 8.25705569695856e-07, 8.340484214386379e-07, 8.428364708379377e-07, 8.566150313527032e-07, 8.733283038964146e-07, 9.080939094019413e-07, 9.449199183109158e-07, 9.813679753278848e-07, 1.0326286883355351e-06, 3.500364073261153e-06, 3.322727934573777e-06, 2.87987791125488e-06, 2.374376435909653e-06, 2.886937636503717e-06, 2.5917670427588746e-06, 2.0112320271437056e-06, 1.235365630236629e-06, 9.258641853193694e-07, 8.803069135865371e-07, 8.08062679880095e-07, 7.582427201668906e-07 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 2.4877786927390844e-05, "first_to_last_ratio": 1.6514628148415365, "imbalance_abs_log_ratio": 0.501661449604626, "last_quartile_mean": 1.506409148532839e-05, "mean": 2.0078249377775137e-05, "normalized": [ 1.4591342632026, 1.5536422632857267, 1.3517883146231473, 1.2162825347932262, 1.1935950051296387, 1.1179709933912532, 1.0509390133477847, 0.9689806884074076, 0.8915451386494992, 0.8246388138987432, 0.7820954831110875, 0.7482109072505639, 0.7604346668089348, 0.728495301919915, 0.7070251881348206, 0.6848417291624026, 0.7463191926383586, 0.726022782634617, 0.7427734847463756, 0.7659822528574189, 2.349755966813442, 1.957225683053433, 1.5220189844214176, 1.1481279918571836, 1.4786737971119177, 1.231344772220105, 0.9170758118251237, 0.6040651256626465, 0.4853764087966196, 0.4560677420977832, 0.423409415130201, 0.4061402830166059 ], "peak_layer": 21, "peak_normalized": 2.349755966813442, "population_cv": 0.44008169029072697, "rest_mean": 1.7508380383164707e-05, "spike_contrast": 1.9393879505451925, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.3955541948671455e-05, "top_five_layers": [ 21, 22, 2, 23, 25 ], "top_five_spike_overlap": 4 }, "values": [ 2.929686161223799e-05, 3.11944168061018e-05, 2.7141542886965908e-05, 2.442082404741086e-05, 2.396529816905968e-05, 2.2446900402428582e-05, 2.1101015590829775e-05, 1.9455435904092155e-05, 1.7900665625347756e-05, 1.6557303752051666e-05, 1.5703108147135936e-05, 1.5022765182948206e-05, 1.526819687569514e-05, 1.4626910342485644e-05, 1.4195828043739311e-05, 1.3750423022429459e-05, 1.4984782865212765e-05, 1.457726648368407e-05, 1.4913591257936787e-05, 1.537958269182127e-05, 4.71789862785954e-05, 3.929766535293311e-05, 3.0559476726921275e-05, 2.3052400138112716e-05, 2.968918124679476e-05, 2.472324740665499e-05, 1.8413276848150417e-05, 1.2128570233471692e-05, 9.745508577907458e-06, 9.157041858998127e-06, 8.501319825882092e-06, 8.154585884767585e-06 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 1.8771140616991033e-05, "first_to_last_ratio": 1.736363274700894, "imbalance_abs_log_ratio": 0.5517928539667487, "last_quartile_mean": 1.0810606795530475e-05, "mean": 1.4905130200304484e-05, "normalized": [ 1.4801939990017345, 1.5691039111063494, 1.3874049931620023, 1.239774027397911, 1.2163430125185601, 1.138365247170121, 1.056427599058293, 0.9873830622053379, 0.9144591875416515, 0.8440609463139155, 0.8074987854330266, 0.7738879258591033, 0.7893782457277657, 0.7619191899697844, 0.7397845827148011, 0.7172839840764536, 0.8053999792272584, 0.7726997047506917, 0.7979343163805397, 0.8279975979222163, 2.3018588110401432, 1.8571207021051657, 1.4114059246003101, 0.9999594243359943, 1.3815470563854542, 1.1024412219823339, 0.8333462709299153, 0.5881170944444888, 0.5059978845852114, 0.4814786280708805, 0.4595441018005674, 0.4498825821820183 ], "peak_layer": 21, "peak_normalized": 2.3018588110401432, "population_cv": 0.4115386118604183, "rest_mean": 1.3275562297268254e-05, "spike_contrast": 1.7855964475100181, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.370479687670013e-05, "top_five_layers": [ 21, 22, 2, 1, 23 ], "top_five_spike_overlap": 3 }, "values": [ 2.206248427683022e-05, 2.338769809284713e-05, 2.0679452063632198e-05, 1.8478993297321722e-05, 1.8129750969819725e-05, 1.696748222457245e-05, 1.574619091115892e-05, 1.4717073099745903e-05, 1.3630133253172971e-05, 1.2580838301801123e-05, 1.2035874533466995e-05, 1.1534900295373518e-05, 1.1765785529860295e-05, 1.1356504728610162e-05, 1.1026585525542032e-05, 1.0691211173252668e-05, 1.2004591553704813e-05, 1.1517189705045894e-05, 1.1893314876942895e-05, 1.2341412002569996e-05, 3.430950528127141e-05, 2.7680625862558372e-05, 2.1037189071648754e-05, 1.4904525414749514e-05, 2.0592138753272593e-05, 1.6432029951829463e-05, 1.2421134670148604e-05, 8.765961865719873e-06, 7.5419643508212175e-06, 7.176501640060451e-06, 6.849564670119435e-06, 6.705558462272165e-06 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 6.533599525937461e-05, "first_to_last_ratio": 1.3715994047154707, "imbalance_abs_log_ratio": 0.31597750759914905, "last_quartile_mean": 4.763489619108441e-05, "mean": 5.6200549295226665e-05, "normalized": [ 1.4017432931406517, 1.4583733820552598, 1.2431667709761864, 1.1144108968642796, 1.0993948653410723, 1.0540632442334565, 1.0078719810197756, 0.9213822106045704, 0.8454719287138224, 0.7873085308710237, 0.7432481184741694, 0.717230415759656, 0.7302510159905761, 0.699039728528033, 0.6804589010122792, 0.6608257239695449, 0.6841580160838986, 0.6687067260124095, 0.6827302204563772, 0.715286964332668, 2.3210225524575905, 2.0221393627018283, 1.6333306175863107, 1.3276826658838428, 1.6116590918780722, 1.4300423487726395, 1.0960444620183518, 0.685528459082958, 0.5442429776147171, 0.5055347435773276, 0.46704520689350315, 0.440604577093148 ], "peak_layer": 21, "peak_normalized": 2.3210225524575905, "population_cv": 0.44945082305341, "rest_mean": 4.804973307020393e-05, "spike_contrast": 2.0856506479219887, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00010021495691034943, "top_five_layers": [ 21, 22, 23, 25, 2 ], "top_five_spike_overlap": 4 }, "values": [ 7.877874304540455e-05, 8.196138514904305e-05, 6.986665539443493e-05, 6.26305045443587e-05, 6.178659532452002e-05, 5.9238933317828923e-05, 5.664295895257965e-05, 5.1782186346827075e-05, 4.7515986807411537e-05, 4.4247171899769455e-05, 4.1770952520892024e-05, 4.030874333693646e-05, 4.104050822206773e-05, 3.928641672246158e-05, 3.824216400971636e-05, 3.713876867550425e-05, 3.845005630864762e-05, 3.7581685319310054e-05, 3.8369813410099596e-05, 4.0199520299211144e-05, 0.00013044274237472564, 0.00011364534293534234, 9.179407788906246e-05, 7.461649511242285e-05, 9.057612624019384e-05, 8.036916551645845e-05, 6.159830081742257e-05, 3.8527075957972556e-05, 3.058675429201685e-05, 2.8411330276867375e-05, 2.624819717311766e-05, 2.476221925462596e-05 ] } } }, "mlp_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "c0b8018c349b1bd09e5d17858eb312a2323da0e92de9651e2bb08ae4adf115c7", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.13583384454250336, 0.0415157787501812, 0.018140379339456558, 0.039518751204013824, 0.022877270355820656, 0.015532293356955051, 0.02314678393304348, 0.05432315170764923, 0.03743875026702881, 0.04006272554397583, 0.045583780854940414, 0.05995568633079529, 0.038651175796985626, 0.04559725895524025, 0.05632574483752251, 0.07952121645212173, 0.042337141931056976, 0.05044656619429588, 0.04418695345520973, 0.06310147047042847, 0.029010163620114326, 0.04586305841803551, 0.06033914536237717, 0.07656749337911606, 0.045884523540735245, 0.051604680716991425, 0.06502768397331238, 0.06792232394218445, 0.060306549072265625, 0.08025261014699936, 0.09663844108581543, 0.12230555713176727 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 7.830868128166912e-06, "first_to_last_ratio": 1.6846032823097203, "imbalance_abs_log_ratio": 0.5215300953180523, "last_quartile_mean": 4.648493927561503e-06, "mean": 6.24525678460941e-06, "normalized": [ 1.3672071926622797, 1.5945192381883362, 1.3831357293784907, 1.212526708567312, 1.227902653490279, 1.1527867226569373, 1.0850341921519409, 1.0080115320133034, 0.8861589615641356, 0.8263664761567352, 0.7864352567781187, 0.7537326574657187, 0.7732400433897239, 0.7433297474226034, 0.7205135825189163, 0.7031152055156547, 0.7492724574784508, 0.7363867071446466, 0.7560912756455532, 0.8011059171959647, 2.357482268387036, 1.833740878351764, 1.447775935332343, 1.1395374078451164, 1.5406369874197603, 1.1553294895579622, 0.7391442049981392, 0.5691842556384004, 0.5336017567325627, 0.49181557939731235, 0.47119405366194494, 0.4536849252925581 ], "peak_layer": 21, "peak_normalized": 2.357482268387036, "population_cv": 0.4260631099292102, "rest_mean": 5.477512685400951e-06, "spike_contrast": 1.897042602572168, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.0391074920335085e-05, "top_five_layers": [ 21, 22, 2, 25, 23 ], "top_five_spike_overlap": 4 }, "values": [ 8.538559995940886e-06, 9.958182090485934e-06, 8.638037797936704e-06, 7.572540653200122e-06, 7.668567377550062e-06, 7.199449100880884e-06, 6.776317150070099e-06, 6.295290859270608e-06, 5.534290266950848e-06, 5.1608708417916205e-06, 4.911490123049589e-06, 4.70725399281946e-06, 4.829082627111347e-06, 4.642285148293013e-06, 4.499792339629494e-06, 4.391135007608682e-06, 4.679398898588261e-06, 4.598924078891287e-06, 4.7219841690093745e-06, 5.003112164558843e-06, 1.4723082131240517e-05, 1.1452182661741972e-05, 9.041732482728548e-06, 7.1167037276609335e-06, 9.62167359830346e-06, 7.21532933312119e-06, 4.616145361069357e-06, 3.5547018342185766e-06, 3.3324799915135372e-06, 3.071514584007673e-06, 2.942727860499872e-06, 2.833378857758362e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 3.0980921792433946e-05, "first_to_last_ratio": 1.6658131427611296, "imbalance_abs_log_ratio": 0.5103133782473317, "last_quartile_mean": 1.859807741766417e-05, "mean": 2.4880517656811207e-05, "normalized": [ 1.370507913630155, 1.5983279408431241, 1.3802618220339473, 1.2031563282890498, 1.209277451205461, 1.1336955543149037, 1.0700398531130362, 0.9962370633835861, 0.88025799237967, 0.8231992073715708, 0.7835572490779874, 0.7540014795271006, 0.7755467706503991, 0.7466935784497585, 0.7242833349195127, 0.7069132248467042, 0.7581130557602427, 0.7413017907722709, 0.7579739293609171, 0.8000455916771936, 2.3697545084000975, 1.8486975573126212, 1.4482913413997927, 1.1399006794556055, 1.5423453201009587, 1.1547938571011154, 0.738521821600251, 0.5737014521081468, 0.5383420354101978, 0.4967444117556274, 0.4763396208054652, 0.45917626294353037 ], "peak_layer": 21, "peak_normalized": 2.3697545084000975, "population_cv": 0.4264500259197146, "rest_mean": 2.1794421728474037e-05, "spike_contrast": 1.9062417065901562, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.154543566983193e-05, "top_five_layers": [ 21, 22, 2, 25, 23 ], "top_five_spike_overlap": 4 }, "values": [ 3.409894634387456e-05, 3.976722655352205e-05, 3.434162863413803e-05, 2.9935152269899845e-05, 3.0087448976701126e-05, 2.820693225658033e-05, 2.6623145458870567e-05, 2.478689384588506e-05, 2.1901274521951564e-05, 2.0481622414081357e-05, 1.9495309970807284e-05, 1.87599471246358e-05, 1.9296005120850168e-05, 1.857812276284676e-05, 1.8020544302999042e-05, 1.7588366972631775e-05, 1.8862245269701816e-05, 1.8443972294335254e-05, 1.885878373286687e-05, 1.9905548469978385e-05, 5.8960718888556585e-05, 4.599655221682042e-05, 3.603423829190433e-05, 2.8361318982206285e-05, 3.837434996967204e-05, 2.873186895158142e-05, 1.8374805222265422e-05, 1.4273989108914975e-05, 1.339422851742711e-05, 1.2359258107608184e-05, 1.1851576346089132e-05, 1.1424543117755093e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.027474191039800644, "first_to_last_ratio": 1.6658131358071762, "imbalance_abs_log_ratio": 0.5103133740728218, "last_quartile_mean": 0.01649296097457409, "mean": 0.022064291697461158, "normalized": [ 1.3705079763477521, 1.5983279764429872, 1.3802618304998915, 1.2031562834588554, 1.2092774197716574, 1.1336954969421926, 1.0700398664707953, 0.9962370737878268, 0.880257964374106, 0.8231992556250316, 0.7835572812965553, 0.7540014342522046, 0.7755467636405156, 0.746693620242164, 0.7242833339152528, 0.7069131987753602, 0.7581130610035737, 0.7413017795561986, 0.7579739385133518, 0.8000455757004906, 2.3697545870144685, 1.8486974697743424, 1.4482912930956124, 1.1399007145658326, 1.5423453565184628, 1.1547938277954763, 0.7385218623213646, 0.5737014727512196, 0.538342031412859, 0.49674440683651866, 0.476339620169168, 0.4591762271279121 ], "peak_layer": 21, "peak_normalized": 2.3697545870144685, "population_cv": 0.4264500304805215, "rest_mean": 0.019327510976129107, "spike_contrast": 1.9062417110075849, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.036842907592654225, "top_five_layers": [ 21, 22, 2, 25, 23 ], "top_five_spike_overlap": 4 }, "values": [ 0.030239287763834, 0.0352659747004509, 0.030454499647021294, 0.026546791195869446, 0.026681849732995033, 0.025014188140630722, 0.023609671741724014, 0.021981265395879745, 0.019422268494963646, 0.01816330850124359, 0.017288636416196823, 0.016636507585644722, 0.017111890017986298, 0.016475265845656395, 0.0159807987511158, 0.015597539022564888, 0.016727227717638016, 0.016356298699975014, 0.016724158078432083, 0.017652438953518867, 0.05228695645928383, 0.04079020023345947, 0.031955521553754807, 0.025151101872324944, 0.03403075784444809, 0.02547970786690712, 0.01629496179521084, 0.012658316642045975, 0.011878135614097118, 0.010960313491523266, 0.010510096326470375, 0.01013139821588993 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 1.1997962658938377e-06, "first_to_last_ratio": 0.956904421295307, "imbalance_abs_log_ratio": 0.044051765771491065, "last_quartile_mean": 1.253830831160485e-06, "mean": 1.2467580212671692e-06, "normalized": [ 1.7326278711879082, 1.0802094091466963, 0.9875616383308882, 0.8642255945421322, 0.7944081458591375, 0.7570699993504425, 0.7435533664527421, 0.7390071988045177, 0.6816364524571962, 0.6498214856567962, 0.6407427370696622, 0.6423419109931766, 0.6585804906451959, 0.6610107334861107, 0.6695214846751223, 0.6893333232838798, 0.7232474366733751, 0.7533514811457979, 0.7833143697388991, 0.8318899558612894, 2.402316408557421, 2.112852752028863, 1.8390615926595408, 1.516930471710572, 2.087753631812459, 1.619633319278821, 1.0594938719965854, 0.7480633332714504, 0.6964698107408152, 0.6444572430916905, 0.6140394277014737, 0.5754730517893415 ], "peak_layer": 21, "peak_normalized": 2.402316408557421, "population_cv": 0.5088534298543446, "rest_mean": 1.0177740629539282e-06, "spike_contrast": 2.4399043820701896, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.4832713961586705e-06, "top_five_layers": [ 21, 22, 25, 23, 1 ], "top_five_spike_overlap": 4 }, "values": [ 2.160167696274584e-06, 1.3467597455019131e-06, 1.2312503940847819e-06, 1.0774801921797916e-06, 9.90434728009859e-07, 9.438830943508947e-07, 9.27031123865163e-07, 9.213631528837141e-07, 8.498357146891067e-07, 8.101701496343594e-07, 7.988511470102821e-07, 8.00844929926825e-07, 8.210905093619658e-07, 8.241204341175035e-07, 8.347312814294128e-07, 8.594318501309317e-07, 9.017145430334494e-07, 9.39247001952026e-07, 9.766034736458096e-07, 1.037165475281654e-06, 2.995107251990703e-06, 2.634216116348398e-06, 2.292864792252658e-06, 1.8912452333097463e-06, 2.6029235868918477e-06, 2.01929083232244e-06, 1.3209324833951541e-06, 9.326539611720364e-07, 8.683293231115385e-07, 8.03482237188291e-07, 7.655585818611144e-07, 7.174756433414586e-07 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 2.4322916033270303e-05, "first_to_last_ratio": 1.933182050307782, "imbalance_abs_log_ratio": 0.659167375981783, "last_quartile_mean": 1.2581803161992866e-05, "mean": 1.8593932111343747e-05, "normalized": [ 1.448768523065353, 1.6536599284285733, 1.4497600979772078, 1.281496311585712, 1.2816500956979295, 1.1961515097924273, 1.1161774126815454, 1.0372193472433668, 0.9201116696923195, 0.8618286650864359, 0.8198341077734796, 0.7875804826525246, 0.8071034343947527, 0.7733668980364898, 0.7480086120692532, 0.7281214502444642, 0.8002546609136906, 0.7794316855917821, 0.7993192387247446, 0.843815380277884, 2.3190988473580583, 1.7733464246705644, 1.3419510765549716, 1.0186499156931126, 1.384285533930205, 1.0159150628679534, 0.6643007012795805, 0.5259497834281985, 0.4931564954223532, 0.4597801093243501, 0.4422749870927152, 0.42763155044800155 ], "peak_layer": 21, "peak_normalized": 2.3190988473580583, "population_cv": 0.4203136767821878, "rest_mean": 1.663996341381722e-05, "spike_contrast": 1.751528074501999, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.9145363077986987e-05, "top_five_layers": [ 21, 22, 2, 3, 1 ], "top_five_spike_overlap": 2 }, "values": [ 2.6938303562928922e-05, 3.074804044445045e-05, 2.6956740839523263e-05, 2.3828055418562144e-05, 2.3830914869904518e-05, 2.224115996796172e-05, 2.0754127035615966e-05, 1.9285986127215438e-05, 1.7108493921114132e-05, 1.6024783690227196e-05, 1.5243939742504153e-05, 1.4644218026660383e-05, 1.5007226465968415e-05, 1.4379931599250995e-05, 1.3908421351516154e-05, 1.353864081465872e-05, 1.4879880836815573e-05, 1.449269984732382e-05, 1.4862487660138868e-05, 1.5689845895394683e-05, 4.312116652727127e-05, 3.297348303021863e-05, 2.4952147214207798e-05, 1.8940707377623767e-05, 2.573931124061346e-05, 1.888985570985824e-05, 1.2351962141110562e-05, 9.77947456703987e-06, 9.169718396151438e-06, 8.549120138923172e-06, 8.223631084547378e-06, 7.95135201769881e-06 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 1.8488175783204497e-05, "first_to_last_ratio": 1.996647913190845, "imbalance_abs_log_ratio": 0.6914697310232422, "last_quartile_mean": 9.259607395506464e-06, "mean": 1.3996616615941093e-05, "normalized": [ 1.4860415947444752, 1.654572193766105, 1.4726422791377596, 1.296721938844898, 1.2919339811945392, 1.204373825532876, 1.110059949031577, 1.0508798983536942, 0.9398717851152208, 0.875184647433028, 0.841455485858673, 0.8076562112841452, 0.8307922667285831, 0.7995797005051963, 0.7744505254552885, 0.750304102040797, 0.852541072459684, 0.8212679452389283, 0.8495607176256446, 0.8998735206394037, 2.265665406942824, 1.7049475071655606, 1.2370709159471331, 0.8900692669859449, 1.241435856330462, 0.9158829372571883, 0.6358974901720724, 0.5404267816577268, 0.513431197243625, 0.4894035565578787, 0.4811959169241117, 0.4748095258249554 ], "peak_layer": 21, "peak_normalized": 2.265665406942824, "population_cv": 0.39867979236658496, "rest_mean": 1.2783996950289563e-05, "spike_contrast": 1.6070688134820006, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.0544762810459362e-05, "top_five_layers": [ 21, 22, 2, 1, 3 ], "top_five_spike_overlap": 2 }, "values": [ 2.079955447698012e-05, 2.3158412659540772e-05, 2.0612009393516928e-05, 1.814971983549185e-05, 1.8082704627886415e-05, 1.685715869825799e-05, 1.5537083527306095e-05, 1.4708763046655804e-05, 1.3155025044397917e-05, 1.2249623978277668e-05, 1.177752983494429e-05, 1.1304454346827697e-05, 1.162828084488865e-05, 1.1191410521860234e-05, 1.08396870928118e-05, 1.050171886163298e-05, 1.1932690540561453e-05, 1.1494972568470985e-05, 1.1890975656569935e-05, 1.2595184671226889e-05, 3.171165008097887e-05, 2.386349660810083e-05, 1.731480733724311e-05, 1.2457958291633986e-05, 1.7375901734340005e-05, 1.2819262337870896e-05, 8.900413376977667e-06, 7.564146471850108e-06, 7.186299626482651e-06, 6.849993951618671e-06, 6.735114766343031e-06, 6.645726898568682e-06 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 6.342507776935236e-05, "first_to_last_ratio": 1.616200925927208, "imbalance_abs_log_ratio": 0.48007828772089967, "last_quartile_mean": 3.924331235793943e-05, "mean": 5.137373779007248e-05, "normalized": [ 1.3515999394598013, 1.5835927674465278, 1.3499070604694168, 1.1745870850418234, 1.1978293660887585, 1.1347989760913781, 1.0860373904914504, 0.9983012069358712, 0.8801381684302598, 0.8330265485492294, 0.7891536768278336, 0.7664748626146008, 0.7807981229621749, 0.7474426850820692, 0.7266842751748783, 0.7092460495035767, 0.7383486122257505, 0.7253378889854624, 0.7431997946952259, 0.7947606568989308, 2.2887095121861414, 1.8252035501088177, 1.4656792774041523, 1.198111630610455, 1.557842963390745, 1.1997538067916234, 0.7510261556931884, 0.5965295974058308, 0.5535161595448382, 0.5114560902878591, 0.48208415118823095, 0.45882197141309805 ], "peak_layer": 21, "peak_normalized": 2.2887095121861414, "population_cv": 0.41529173707248473, "rest_mean": 4.502708913976137e-05, "spike_contrast": 1.9020914329130534, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 8.564564050175249e-05, "top_five_layers": [ 21, 22, 2, 25, 23 ], "top_five_spike_overlap": 4 }, "values": [ 6.943674088688567e-05, 8.135507960105315e-05, 6.934977136552334e-05, 6.03429289185442e-05, 6.153697177069262e-05, 5.8298865042161196e-05, 5.5793800129322335e-05, 5.128646444063634e-05, 4.521598748397082e-05, 4.2795687477337196e-05, 4.054177406942472e-05, 3.9376678614644334e-05, 4.0112518036039546e-05, 3.8398924516513944e-05, 3.733248740900308e-05, 3.6436620575841516e-05, 3.793172800214961e-05, 3.7263318517943844e-05, 3.8180951378308237e-05, 4.082982559339143e-05, 0.00011757956235669553, 9.376752859679982e-05, 7.529742288170382e-05, 6.155147275421768e-05, 8.003221591934562e-05, 6.163583748275414e-05, 3.858302079606801e-05, 3.064595512114465e-05, 2.8436194043024443e-05, 2.627541107358411e-05, 2.4766464775893837e-05, 2.357139965170063e-05 ] } } }, "post_attention_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "90a633f12d984935028fe5d7212be48e7da7b4a0806751939a551938d16dc540", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.0933690071105957, 0.35791492462158203, 0.3940892517566681, 0.4057151973247528, 0.09380833059549332, 0.11286329478025436, 0.13231655955314636, 0.1621972620487213, 0.05156813934445381, 0.08533531427383423, 0.12003090232610703, 0.16063299775123596, 0.046871740370988846, 0.079384446144104, 0.11459959298372269, 0.15625107288360596, 0.08397205173969269, 0.11654024571180344, 0.15294702351093292, 0.21572710573673248, 0.03708760067820549, 0.06553087383508682, 0.09390194714069366, 0.12852425873279572, 0.025487789884209633, 0.06059965491294861, 0.08860157430171967, 0.12100546807050705, 0.024166326969861984, 0.06877554953098297, 0.12682415544986725, 0.2025238573551178 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 8.033873655222123e-06, "first_to_last_ratio": 1.425887348236832, "imbalance_abs_log_ratio": 0.3547943204401696, "last_quartile_mean": 5.634297593815063e-06, "mean": 6.824663053350832e-06, "normalized": [ 1.3817351206048945, 1.4804623464257967, 1.2768988537517754, 1.1433247335509922, 1.1306478170747023, 1.0654694093662715, 1.0110988642200183, 0.9278225507220519, 0.8519651485537575, 0.7842382974941985, 0.7445200689075784, 0.7100674416225211, 0.7217466631164221, 0.6945954732911409, 0.6750206484282966, 0.6552172059547812, 0.6913487167485722, 0.6794550038268806, 0.6949213083585211, 0.7189009702283647, 2.40033965224001, 2.041141472745998, 1.6243568426135113, 1.2900744511369144, 1.5951928049936666, 1.3978349711121278, 1.051346824664649, 0.6710026156270521, 0.5205928506999057, 0.48847250951473326, 0.4495528433749468, 0.43063551902894753 ], "peak_layer": 21, "peak_normalized": 2.40033965224001, "population_cv": 0.45909247198274766, "rest_mean": 5.825960762969636e-06, "spike_contrast": 2.09710568239077, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.221765542140929e-05, "top_five_layers": [ 21, 22, 23, 25, 2 ], "top_five_spike_overlap": 4 }, "values": [ 9.42987662710948e-06, 1.0103656677529216e-05, 8.71440443006577e-06, 7.80280606704764e-06, 7.716290383541491e-06, 7.271469712577527e-06, 6.900409061927348e-06, 6.332076281978516e-06, 5.814375072077382e-06, 5.352162133931415e-06, 5.081098606751766e-06, 4.845971034228569e-06, 4.925677785649896e-06, 4.740380063594785e-06, 4.606788479577517e-06, 4.471636657399358e-06, 4.718222044175491e-06, 4.6370514610316604e-06, 4.742603778140619e-06, 4.9062568905355874e-06, 1.638150934013538e-05, 1.3930102795711718e-05, 1.1085688129242044e-05, 8.804323442745954e-06, 1.0886653399211355e-05, 9.539752682030667e-06, 7.175087830546545e-06, 4.5793667595717125e-06, 3.552870794010232e-06, 3.333660288262763e-06, 3.068046680709813e-06, 2.9389423161774175e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 3.18315117056045e-05, "first_to_last_ratio": 1.413122720786904, "imbalance_abs_log_ratio": 0.3458019511602721, "last_quartile_mean": 2.2525652752847236e-05, "mean": 2.719590119681925e-05, "normalized": [ 1.3914492796778657, 1.4843315127212897, 1.2741938645217614, 1.1347206932524478, 1.1138340093422883, 1.0480527835825995, 0.9970527397490913, 0.9199854583906706, 0.8457241904051724, 0.7806623763642918, 0.74049196700929, 0.7096220074798465, 0.7229446349850482, 0.6960823391576559, 0.6775035114540296, 0.6578212870299519, 0.6991469956243525, 0.6834050820201355, 0.6966778804317604, 0.7180997756262174, 2.404657347296138, 2.0537854838940115, 1.635716044550207, 1.2878482400862126, 1.5982537962675871, 1.3983543212385119, 1.0506101865651531, 0.672302089816195, 0.5245715354519678, 0.49287304183301744, 0.4541373425072359, 0.43508818166799623 ], "peak_layer": 21, "peak_normalized": 2.404657347296138, "population_cv": 0.4601693450991735, "rest_mean": 2.31867611041202e-05, "spike_contrast": 2.106601153910819, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.8845257697394115e-05, "top_five_layers": [ 21, 22, 23, 25, 2 ], "top_five_spike_overlap": 4 }, "values": [ 3.784171713050455e-05, 4.036773316329345e-05, 3.4652850445127115e-05, 3.085975185967982e-05, 3.029171966772992e-05, 2.8502739951363765e-05, 2.7115747798234224e-05, 2.5019833628903143e-05, 2.300023152201902e-05, 2.1230816855677404e-05, 2.013834637182299e-05, 1.9298810002510436e-05, 1.9661130863823928e-05, 1.8930586520582438e-05, 1.842531855800189e-05, 1.7890042727231048e-05, 1.901393261505291e-05, 1.858581708802376e-05, 1.894678280223161e-05, 1.952937054738868e-05, 6.539682362927124e-05, 5.585454709944315e-05, 4.448477193363942e-05, 3.5024193493882194e-05, 4.346595233073458e-05, 3.8029505958547816e-05, 2.8572290830197744e-05, 1.828386120905634e-05, 1.4266195648815483e-05, 1.3404126548266504e-05, 1.2350674296612851e-05, 1.183261520054657e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.028228502487763762, "first_to_last_ratio": 1.4131227175222638, "imbalance_abs_log_ratio": 0.34580194885004095, "last_quartile_mean": 0.019975973875261843, "mean": 0.02411759688402526, "normalized": [ 1.39144930349893, 1.4843314964609153, 1.274193916464304, 1.1347206428874501, 1.1138340006889969, 1.0480527513487479, 0.9970527396058958, 0.919985454502784, 0.8457242319826253, 0.780662396755172, 0.7404919809032963, 0.7096220487556825, 0.7229446867895516, 0.696082389395248, 0.6775035102783245, 0.6578212978264142, 0.699147024007148, 0.6834051001887027, 0.6966778464869985, 0.7180997822137167, 2.4046573114569996, 2.053785417235791, 1.6357159971310755, 1.2878481877993373, 1.5982537911857264, 1.3983542888095704, 1.050610204803511, 0.6723021037653274, 0.5245715162557125, 0.4928730414777099, 0.45413736056661846, 0.4350881784717163 ], "peak_layer": 21, "peak_normalized": 2.4046573114569996, "population_cv": 0.4601693286164402, "rest_mean": 0.020562251581362, "spike_contrast": 2.10660108631633, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.04331646151840687, "top_five_layers": [ 21, 22, 23, 25, 2 ], "top_five_spike_overlap": 4 }, "values": [ 0.03355841338634491, 0.035798508673906326, 0.03073049522936344, 0.02736673504114151, 0.02686299942433834, 0.025276513770222664, 0.024046516045928, 0.022187838330864906, 0.02039683610200882, 0.01882770098745823, 0.01785888709127903, 0.01711437851190567, 0.01743568852543831, 0.016787834465503693, 0.016339756548404694, 0.01586506888270378, 0.016861746087670326, 0.016482088714838028, 0.016802195459604263, 0.017318841069936752, 0.05799455568194389, 0.049532368779182434, 0.0394495390355587, 0.031059803441166878, 0.038546040654182434, 0.03372494503855705, 0.025338193401694298, 0.016214311122894287, 0.012651404365897179, 0.01188691332936287, 0.010952701792120934, 0.010493281297385693 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 1.2542172598273282e-06, "first_to_last_ratio": 0.8293745988896173, "imbalance_abs_log_ratio": 0.18708335752548239, "last_quartile_mean": 1.512244601542534e-06, "mean": 1.4018573466501039e-06, "normalized": [ 1.7194707335121522, 0.9885732594586992, 0.9006271365272239, 0.806722957582638, 0.7084104108205443, 0.6842793432388339, 0.6715264693306607, 0.6778498281056683, 0.6320305867365923, 0.6003501720342246, 0.587164887353172, 0.5889721811039229, 0.5949683933939726, 0.6012316996225965, 0.6111086201851915, 0.6229508120277141, 0.6477696252132137, 0.6740517174273047, 0.7000730412917721, 0.736640135653066, 2.4970937111977958, 2.3702444052471936, 2.0542439324094195, 1.693697364624819, 2.059431564127265, 1.8488119971201133, 1.4347822503690206, 0.8812629679008496, 0.6604335759382544, 0.6279596242117634, 0.5764177928233983, 0.5408488034109441 ], "peak_layer": 21, "peak_normalized": 2.4970937111977958, "population_cv": 0.5863938469763146, "rest_mean": 1.107222706869598e-06, "spike_contrast": 2.7030554764600927, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.992884401464835e-06, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 2.4104526801238535e-06, 1.3858386864740169e-06, 1.262550767933135e-06, 1.1309105047985213e-06, 9.930903388521983e-07, 9.592620244802674e-07, 9.413843145011924e-07, 9.502487614554411e-07, 8.860167213242676e-07, 8.416052992288314e-07, 8.231214110310248e-07, 8.256549790530698e-07, 8.340608133039495e-07, 8.428410751548654e-07, 8.566871088078187e-07, 8.732881724426989e-07, 9.080806080419279e-07, 9.44924352097587e-07, 9.814025361265521e-07, 1.0326643860025797e-06, 3.5005691643164027e-06, 3.3227445328520844e-06, 2.8797569484595442e-06, 2.374322093601222e-06, 2.8870292680949206e-06, 2.5917706807376817e-06, 2.01136003852298e-06, 1.2354049658824806e-06, 9.258336604034412e-07, 8.80309812600899e-07, 8.080555176093185e-07, 7.581928684885497e-07 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 2.4877815349100274e-05, "first_to_last_ratio": 1.6514611868893043, "imbalance_abs_log_ratio": 0.501660463840357, "last_quartile_mean": 1.5064123545016628e-05, "mean": 2.007825864325241e-05, "normalized": [ 1.4591350393782325, 1.5536487939275285, 1.351793760679405, 1.2162843289869831, 1.1935969003879465, 1.117967487848945, 1.0509391625377287, 0.9689743525800812, 0.8915464485343808, 0.8246368026445716, 0.7820986554030648, 0.7482139592867272, 0.7604320510180863, 0.7284932444379358, 0.7070277609045024, 0.684842998541861, 0.7463219737625534, 0.7260248936626227, 0.7427698352633515, 0.765980631051296, 2.349738394189828, 1.9572206124870826, 1.522028609885614, 1.148119942649472, 1.4786839861484071, 1.2313475560078786, 0.9170724895845254, 0.604059456504617, 0.4853766377857473, 0.4560714272210983, 0.423409219740233, 0.40614258695769245 ], "peak_layer": 21, "peak_normalized": 2.349738394189828, "population_cv": 0.44008063750335685, "rest_mean": 1.7508396551959407e-05, "spike_contrast": 1.9393845596006103, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.395551393623464e-05, "top_five_layers": [ 21, 22, 2, 23, 25 ], "top_five_spike_overlap": 4 }, "values": [ 2.9296890716068447e-05, 3.119456232525408e-05, 2.7141664759255946e-05, 2.4420871341135353e-05, 2.3965347281773575e-05, 2.2446840375778265e-05, 2.11010283237556e-05, 1.9455317669780925e-05, 1.7900700186146423e-05, 1.6557271010242403e-05, 1.5703179087722674e-05, 1.5022833395050839e-05, 1.526815140096005e-05, 1.4626875781686977e-05, 1.4195886251400225e-05, 1.3750454854744021e-05, 1.4984845620347187e-05, 1.4577315596397966e-05, 1.4913524864823557e-05, 1.537955722596962e-05, 4.717865522252396e-05, 3.9297581679420546e-05, 3.055968409171328e-05, 2.3052249161992222e-05, 2.9689399525523186e-05, 2.4723314709262922e-05, 1.8413218640489504e-05, 1.212846200360218e-05, 9.745517672854476e-06, 9.15712007554248e-06, 8.501319825882092e-06, 8.154635906976182e-06 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 1.8770221458908054e-05, "first_to_last_ratio": 1.7363166141968762, "imbalance_abs_log_ratio": 0.5517659810584972, "last_quartile_mean": 1.0810367939484422e-05, "mean": 1.4904660503134437e-05, "normalized": [ 1.4804758192761516, 1.5686091752428721, 1.3871683853935661, 1.2396526121725782, 1.2165360924906854, 1.1383324115279394, 1.0565912312145647, 0.987454268728218, 0.914484588132905, 0.8440176157133784, 0.8075967862626621, 0.7737406621473994, 0.7893125668477194, 0.7618917601100988, 0.7397789109815618, 0.7173367324527048, 0.8052285069675325, 0.7726947651821442, 0.79797374091157, 0.8279346615915875, 2.302477120808428, 1.8570723178847308, 1.411236707926337, 0.9999930722292978, 1.381736067340189, 1.102558463858359, 0.8331937415186891, 0.5881840175544256, 0.5060076977276765, 0.48138896734037795, 0.4594751681477022, 0.4498653643159486 ], "peak_layer": 21, "peak_normalized": 2.302477120808428, "population_cv": 0.4115922631993492, "rest_mean": 1.3274799837543267e-05, "spike_contrast": 1.7857827151775683, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.3705908097326757e-05, "top_five_layers": [ 21, 22, 2, 1, 23 ], "top_five_spike_overlap": 3 }, "values": [ 2.206598946941085e-05, 2.337958721909672e-05, 2.0675273844972253e-05, 1.847660132625606e-05, 1.813205744838342e-05, 1.6966458133538254e-05, 1.5748133591841906e-05, 1.4717670637764968e-05, 1.3630082321469672e-05, 1.2579796020872891e-05, 1.2036955922667403e-05, 1.153234188677743e-05, 1.1764435839722864e-05, 1.1355738024576567e-05, 1.102615351555869e-05, 1.0691660463635344e-05, 1.2001657523796894e-05, 1.1516753147589043e-05, 1.189352769870311e-05, 1.2340085049800109e-05, 3.431763980188407e-05, 2.7679032427840866e-05, 2.1034004021203145e-05, 1.4904557247064076e-05, 2.059430698864162e-05, 1.643325958866626e-05, 1.2418469850672409e-05, 8.76668309501838e-06, 7.541872946603689e-06, 7.174939128162805e-06, 6.848321390862111e-06, 6.705090527248103e-06 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 6.533271107400651e-05, "first_to_last_ratio": 1.371465356231375, "imbalance_abs_log_ratio": 0.31587977131081135, "last_quartile_mean": 4.763715742228669e-05, "mean": 5.620124170491181e-05, "normalized": [ 1.4012102444951349, 1.458297285928318, 1.2433484969748743, 1.11440946606093, 1.0993312185756143, 1.0539253265921091, 1.007831211546201, 0.9214713219656424, 0.8455131031805617, 0.7872744273885365, 0.7432122922289986, 0.7172423580920736, 0.7305403009418301, 0.6990685309142981, 0.6803699271783116, 0.660850401237905, 0.6839978569897706, 0.6686896192353879, 0.6826094355765389, 0.7153145308493645, 2.320580971408291, 2.022116909360114, 1.633711181328826, 1.328143378186385, 1.6118222960248818, 1.429782376441371, 1.0961433320653418, 0.6855727045147141, 0.5442685086224606, 0.5056005935859182, 0.46709159386335247, 0.4406587986459428 ], "peak_layer": 21, "peak_normalized": 2.320580971408291, "population_cv": 0.4494140631067826, "rest_mean": 4.8049200106308694e-05, "spike_contrast": 2.0858259058553985, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00010022226633736864, "top_five_layers": [ 21, 22, 23, 25, 2 ], "top_five_spike_overlap": 4 }, "values": [ 7.874975563026965e-05, 8.195811824407429e-05, 6.987772940192372e-05, 6.263119576033205e-05, 6.178377952892333e-05, 5.9231912018731236e-05, 5.664136551786214e-05, 5.178783248993568e-05, 4.751888627652079e-05, 4.424580038175918e-05, 4.17694536736235e-05, 4.0309911128133535e-05, 4.105727202841081e-05, 3.928851947421208e-05, 3.823763472610153e-05, 3.714061313075945e-05, 3.8441528886323795e-05, 3.758118691621348e-05, 3.8363497878890485e-05, 4.020156484330073e-05, 0.0001304195320699364, 0.00011364548117853701, 9.181659697787836e-05, 7.464330701623112e-05, 9.05864144442603e-05, 8.03555449238047e-05, 6.160461634863168e-05, 3.853003727272153e-05, 3.058856600546278e-05, 2.841538116626907e-05, 2.6251127565046772e-05, 2.4765571652096696e-05 ] } } }, "post_mlp_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "89f2226456bfe96e245d2aad0f699d32c676f9f53099789cd870253d002402f5", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.16915453970432281, 0.374446302652359, 0.39966297149658203, 0.4194087088108063, 0.09892481565475464, 0.11664740741252899, 0.13951337337493896, 0.18593810498714447, 0.06607726216316223, 0.10425819456577301, 0.14551182091236115, 0.19865906238555908, 0.06484340131282806, 0.10430748760700226, 0.14720691740512848, 0.20439592003822327, 0.10052042454481125, 0.14085541665554047, 0.1745898574590683, 0.2420409768819809, 0.049456026405096054, 0.08509506285190582, 0.11783231794834137, 0.15649692714214325, 0.05533855780959129, 0.08507220447063446, 0.11942370235919952, 0.1559232622385025, 0.06628894805908203, 0.12384890764951706, 0.19928614795207977, 0.30097496509552 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 7.830877279957349e-06, "first_to_last_ratio": 1.6846144077999308, "imbalance_abs_log_ratio": 0.5215366995159604, "last_quartile_mean": 4.64846866066182e-06, "mean": 6.2452611686580894e-06, "normalized": [ 1.3672341937876606, 1.594431178014059, 1.3831745153175692, 1.2125353961346346, 1.227893053752895, 1.1527838746096812, 1.085041148846834, 1.0080352901771759, 0.8861660578654706, 0.8263696824327974, 0.7864261125716057, 0.7537268856958996, 0.7732384811836543, 0.7433386915420419, 0.7205076156238689, 0.7031265807528125, 0.7492675626170262, 0.7363888843629317, 0.7561041428050577, 0.8011004762449893, 2.3574910988106184, 1.8337474550990784, 1.447778851020939, 1.1395380642083075, 1.5406216342254366, 1.1553109117321185, 0.7391375696916994, 0.5691889531173825, 0.5335968312307019, 0.49181956663231563, 0.47119463307794046, 0.4536846068147965 ], "peak_layer": 21, "peak_normalized": 2.3574910988106184, "population_cv": 0.4260614168253588, "rest_mean": 5.477515691786215e-06, "spike_contrast": 1.897043719901728, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.0391086743766208e-05, "top_five_layers": [ 21, 22, 2, 25, 23 ], "top_five_spike_overlap": 4 }, "values": [ 8.538734618923627e-06, 9.957639122148976e-06, 8.638286089990288e-06, 7.572600225103088e-06, 7.668512807867955e-06, 7.199436367955059e-06, 6.776365353289293e-06, 6.295443654380506e-06, 5.5343384701700415e-06, 5.1608944886538666e-06, 4.911436462862184e-06, 4.707221251010196e-06, 4.829076260648435e-06, 4.642344265448628e-06, 4.499758233578177e-06, 4.391209131426876e-06, 4.6793716137472074e-06, 4.5989409045432694e-06, 4.7220678425219376e-06, 5.003081696486333e-06, 1.4723147614859045e-05, 1.1452231774455868e-05, 9.041757039085496e-06, 7.116712822607951e-06, 9.621584467822686e-06, 7.2152183747675736e-06, 4.6161071622918826e-06, 3.5547336665331386e-06, 3.332451569804107e-06, 3.0715416414750507e-06, 2.942733544841758e-06, 2.833378857758362e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 3.098115166721982e-05, "first_to_last_ratio": 1.665832335654677, "imbalance_abs_log_ratio": 0.5103248998174819, "last_quartile_mean": 1.859800113379606e-05, "mean": 2.488056986749143e-05, "normalized": [ 1.3705174661866055, 1.5983156675512393, 1.3802927019012459, 1.2031581169444034, 1.2092921672792036, 1.1336977811656817, 1.0700344640060477, 0.9962485710707428, 0.8802554872210856, 0.8231957253156242, 0.7835544350794417, 0.7539981426792606, 0.7755501146033489, 0.7467008577178438, 0.7242856167050062, 0.7069129842734175, 0.7581072976930001, 0.741294167153027, 0.7579827202432009, 0.8000412077931803, 2.3697540683317273, 1.8486903149033629, 1.4483038013024037, 1.1398884177364823, 1.5423381356864923, 1.1547708902398965, 0.7385169088414244, 0.5737002847776349, 0.5383410153885547, 0.49674728068386476, 0.47633789014134115, 0.4591752993842084 ], "peak_layer": 21, "peak_normalized": 2.3697540683317273, "population_cv": 0.4264503536406935, "rest_mean": 2.1794480980369606e-05, "spike_contrast": 1.9062371751531715, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.154544985794928e-05, "top_five_layers": [ 21, 22, 2, 25, 23 ], "top_five_spike_overlap": 4 }, "values": [ 3.409925557207316e-05, 3.976700463681482e-05, 3.434246900724247e-05, 2.9935259590274654e-05, 3.0087878258200362e-05, 2.8207046852912754e-05, 2.6623067242326215e-05, 2.478723217791412e-05, 2.1901258151046932e-05, 2.0481578758335672e-05, 1.9495280866976827e-05, 1.8759903468890116e-05, 1.929612881212961e-05, 1.857834286056459e-05, 1.8020638890448026e-05, 1.7588397895451635e-05, 1.8862141587305814e-05, 1.844382131821476e-05, 1.8859042029362172e-05, 1.9905481167370453e-05, 5.8960831665899605e-05, 4.5996468543307856e-05, 3.603462391765788e-05, 2.8361073418636806e-05, 3.837425174424425e-05, 2.8731357815559022e-05, 1.837472154875286e-05, 1.4273990018409677e-05, 1.3394231245911215e-05, 1.2359355423541274e-05, 1.1851558156195097e-05, 1.1424543117755093e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.02747439523227513, "first_to_last_ratio": 1.6658323361770393, "imbalance_abs_log_ratio": 0.5103249001310562, "last_quartile_mean": 0.01649289345368743, "mean": 0.02206433823448606, "normalized": [ 1.370517495304128, 1.598315656933933, 1.3802926868425236, 1.203158135598421, 1.2092921750785575, 1.133697748841484, 1.0700344860990312, 0.9962485640014386, 0.8802555168453254, 0.8231956621595627, 0.783554446792473, 0.7539981555730193, 0.7755501086048634, 0.7467009093273549, 0.7242856051386014, 0.7069130162800921, 0.7581073255068929, 0.7412941378844509, 0.7579827233452033, 0.8000411868815456, 2.369753978621852, 1.848690362677171, 1.4483037714911375, 1.13988843334228, 1.542338220213741, 1.1547708783901962, 0.7385169279147621, 0.5737002627280846, 0.5383409803869399, 0.4967472846020756, 0.47633789793702447, 0.4591752586558334 ], "peak_layer": 21, "peak_normalized": 2.369753978621852, "population_cv": 0.4264503519323746, "rest_mean": 0.019327563716581574, "spike_contrast": 1.9062371839220407, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.03684292063117027, "top_five_layers": [ 21, 22, 2, 25, 23 ], "top_five_spike_overlap": 4 }, "values": [ 0.030239561572670937, 0.03526577726006508, 0.030455244705080986, 0.026546888053417206, 0.026682231575250626, 0.02501429058611393, 0.023609602823853493, 0.02198156528174877, 0.0194222554564476, 0.018163267523050308, 0.017288610339164734, 0.016636470332741737, 0.017111999914050102, 0.016475461423397064, 0.015980882570147514, 0.015597567893564701, 0.016727136448025703, 0.01635616458952427, 0.01672438718378544, 0.017652379348874092, 0.05228705331683159, 0.0407901294529438, 0.031955864280462265, 0.025150883942842484, 0.034030672162771225, 0.025479255244135857, 0.01629488728940487, 0.012658316642045975, 0.011878137476742268, 0.010960400104522705, 0.010510080493986607, 0.01013139821588993 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 1.199762493797607e-06, "first_to_last_ratio": 0.9568845898246269, "imbalance_abs_log_ratio": 0.044072490595978564, "last_quartile_mean": 1.2538215230506466e-06, "mean": 1.2467440040353495e-06, "normalized": [ 1.7326241897503085, 1.0801617353502706, 0.9875048072465176, 0.8641741246193572, 0.7944106943570564, 0.7570867635654143, 0.7435549784387712, 0.7390157810761795, 0.6816481739657032, 0.6498756161674363, 0.6407548650836493, 0.6423202722056006, 0.6586101447391347, 0.6609660975035254, 0.6695103644127346, 0.6893480037191584, 0.7232140781137624, 0.7533838421226347, 0.783337310582298, 0.8318559038504049, 2.402219403660741, 2.1129146231465454, 1.8391853106632379, 1.5169084986348003, 2.0877554020710325, 1.6197058763801093, 1.0594777895547205, 0.7480273813284306, 0.696442670988497, 0.6444830453237016, 0.6140427295043308, 0.5754795218739354 ], "peak_layer": 21, "peak_normalized": 2.402219403660741, "population_cv": 0.508856873306637, "rest_mean": 1.0177594626165281e-06, "spike_contrast": 2.43992870507226, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.4832605276969845e-06, "top_five_layers": [ 21, 22, 25, 23, 1 ], "top_five_spike_overlap": 4 }, "values": [ 2.160138819817803e-06, 1.3466851669363678e-06, 1.2311656973906793e-06, 1.0774039083116804e-06, 9.904267699312186e-07, 9.438933830097085e-07, 9.270227110391716e-07, 9.213634939442272e-07, 8.498407737533853e-07, 8.102285278255295e-07, 7.988572860995191e-07, 8.008089480426861e-07, 8.211182489503699e-07, 8.240555189331644e-07, 8.347080324710987e-07, 8.594404903305985e-07, 9.016628155222861e-07, 9.392767879035091e-07, 9.766210951056564e-07, 1.0371113603468984e-06, 2.9949526378914015e-06, 2.6342636374465656e-06, 2.2929932583792834e-06, 1.8911965753432014e-06, 2.60289652942447e-06, 2.019358589677722e-06, 1.3208975815359736e-06, 9.325986525254848e-07, 8.682857242092723e-07, 8.035053724597674e-07, 7.655540912310244e-07, 7.174756433414586e-07 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 2.432301744192955e-05, "first_to_last_ratio": 1.9331908439082446, "imbalance_abs_log_ratio": 0.6591719247414103, "last_quartile_mean": 1.2581798387145682e-05, "mean": 1.859397696080123e-05, "normalized": [ 1.4487770612718216, 1.653654961462933, 1.4497733294787125, 1.2814979162492595, 1.2816563956773, 1.196145591993453, 1.1161731551855811, 1.0372232041689922, 0.9201054394450198, 0.8618263906656938, 0.8198321792124014, 0.7875803438576819, 0.8071054496128935, 0.7733698261564439, 0.7480089600306629, 0.7281196450713806, 0.8002529752326797, 0.779427946860307, 0.7993230825146315, 0.8438106058089522, 2.3191089058829135, 1.7733468429708727, 1.3419547854156444, 1.0186458934375888, 1.3842790645172136, 1.0159100689409357, 0.6642997837458937, 0.5259475365470405, 0.49315608852252424, 0.4597820818597483, 0.44227396922017687, 0.4276305189826469 ], "peak_layer": 21, "peak_normalized": 2.3191089058829135, "population_cv": 0.42031537321452345, "rest_mean": 1.6640001006264896e-05, "spike_contrast": 1.7515291678362448, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.914544711529743e-05, "top_five_layers": [ 21, 22, 2, 3, 1 ], "top_five_spike_overlap": 2 }, "values": [ 2.693852729862556e-05, 3.074802225455642e-05, 2.695705188671127e-05, 2.3828142730053514e-05, 2.383108949288726e-05, 2.224110357929021e-05, 2.075409793178551e-05, 1.9286104361526668e-05, 1.7108419342548586e-05, 1.602478005224839e-05, 1.5243940651998855e-05, 1.4644250768469647e-05, 1.5007300135039259e-05, 1.4380020729731768e-05, 1.3908461369283032e-05, 1.3538639905164018e-05, 1.4879885384289082e-05, 1.4492665286525153e-05, 1.4862594980513677e-05, 1.5689794963691384e-05, 4.312145756557584e-05, 3.297357034171e-05, 2.495227636245545e-05, 1.894067827379331e-05, 2.5739253032952547e-05, 1.8889808416133747e-05, 1.2351974874036387e-05, 9.779456377145834e-06, 9.169732948066667e-06, 8.549177437089384e-06, 8.22363199404208e-06, 7.95135201769881e-06 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 1.8489471813154523e-05, "first_to_last_ratio": 1.9968734559150145, "imbalance_abs_log_ratio": 0.6915826853327667, "last_quartile_mean": 9.259210571599397e-06, "mean": 1.3996629050438969e-05, "normalized": [ 1.4865976691637304, 1.654475463831894, 1.4725928859843, 1.2967553559693688, 1.2922803441007265, 1.2042403272470203, 1.1101767058132026, 1.0508382875597613, 0.939646445784088, 0.8750539108193341, 0.841526410763236, 0.8076491257723509, 0.8308714535095223, 0.7993613086615595, 0.774389016577603, 0.7503346256619167, 0.852697760526099, 0.8211594145520545, 0.8494750995859626, 0.8999062506481139, 2.2654882092676742, 1.705064515208281, 1.2369375185725398, 0.8902301453835567, 1.241275553543071, 0.9159659472174196, 0.6358533239654343, 0.5402688885791327, 0.5134054640690997, 0.489453610888337, 0.48121985676579426, 0.47480910400781506 ], "peak_layer": 21, "peak_normalized": 2.2654882092676742, "population_cv": 0.39869293120406385, "rest_mean": 1.278410836339053e-05, "spike_contrast": 1.6070139720758685, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.0544240760500544e-05, "top_five_layers": [ 21, 22, 2, 1, 3 ], "top_five_spike_overlap": 2 }, "values": [ 2.0807356122531928e-05, 2.3157079340307973e-05, 2.0611336367437616e-05, 1.8150203686673194e-05, 1.8087568605551496e-05, 1.6855305148055777e-05, 1.5538731531705707e-05, 1.4708193702972494e-05, 1.3151882740203291e-05, 1.2247804988874123e-05, 1.1778533007600345e-05, 1.1304365216346923e-05, 1.162939952337183e-05, 1.1188363714609295e-05, 1.0838835805770941e-05, 1.0502155419089831e-05, 1.1934894246223848e-05, 1.1493463716760743e-05, 1.188978785648942e-05, 1.2595653970493004e-05, 3.170919808326289e-05, 2.3865155526436865e-05, 1.73129556060303e-05, 1.2460221114451997e-05, 1.737367347232066e-05, 1.2820435586036183e-05, 8.899803106032778e-06, 7.5619432209350634e-06, 7.185945833043661e-06, 6.850700629001949e-06, 6.735455826856196e-06, 6.645726898568682e-06 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 6.342557708194363e-05, "first_to_last_ratio": 1.6163197346939728, "imbalance_abs_log_ratio": 0.48015179615551884, "last_quartile_mean": 3.924073666894401e-05, "mean": 5.1374555994243565e-05, "normalized": [ 1.351460439326671, 1.583819782124855, 1.3500985665656522, 1.1742391693240029, 1.1976846671566996, 1.134745638195829, 1.0860814887276113, 0.9984444950286706, 0.8799124207010719, 0.8331985279774058, 0.7891486147290685, 0.766521076144631, 0.7807770486059202, 0.7474261074726405, 0.7267094536872926, 0.7091935407920258, 0.7385321549436045, 0.7253208844812393, 0.7433176166385337, 0.7946866045974246, 2.288678443396367, 1.8254482439395896, 1.4657863718978748, 1.1982361576445748, 1.5576544334769562, 1.1995838678352833, 0.7510428030527292, 0.596547147433896, 0.5533328966467058, 0.5115249890913078, 0.4820316842655173, 0.458814664098348 ], "peak_layer": 21, "peak_normalized": 2.288678443396367, "population_cv": 0.41530456795563125, "rest_mean": 4.5027317793466395e-05, "spike_contrast": 1.902170648300715, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 8.564964227844029e-05, "top_five_layers": [ 21, 22, 2, 25, 23 ], "top_five_spike_overlap": 4 }, "values": [ 6.943068001419306e-05, 8.136803808156401e-05, 6.936071440577507e-05, 6.0326015955070034e-05, 6.153051799628884e-05, 5.8297053328715265e-05, 5.5796954256948084e-05, 5.129464261699468e-05, 4.520510992733762e-05, 4.280520442989655e-05, 4.054215969517827e-05, 3.9379679947160184e-05, 4.011207420262508e-05, 3.8398684409912676e-05, 3.733437552000396e-05, 3.643450327217579e-05, 3.794176154769957e-05, 3.72630383935757e-05, 3.818761251750402e-05, 4.0826671465765685e-05, 0.00011757983884308487, 9.378159302286804e-05, 7.530412403866649e-05, 6.155885057523847e-05, 8.002380491234362e-05, 6.162808858789504e-05, 3.858449053950608e-05, 3.064734482904896e-05, 2.8427231882233173e-05, 2.6279369194526225e-05, 2.4764163754298352e-05, 2.357139965170063e-05 ] } } }, "pre_attention_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "ee695e78937de01b7128d3866b8fc5dd328ebdf2c159be37918d4f02952d5248", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.0626930519938469, 0.09030644595623016, 0.20229491591453552, 0.20300470292568207, 0.19049881398677826, 0.12695962190628052, 0.12132706493139267, 0.11585728824138641, 0.13427142798900604, 0.10485932976007462, 0.10419609397649765, 0.10852954536676407, 0.1286265105009079, 0.09096142649650574, 0.10080574452877045, 0.10566592961549759, 0.1287592649459839, 0.09650235623121262, 0.1065765842795372, 0.11446042358875275, 0.12395420670509338, 0.07221997529268265, 0.07677769660949707, 0.08416415005922318, 0.09094838052988052, 0.059439923614263535, 0.062182098627090454, 0.06896103918552399, 0.0776243805885315, 0.06285123527050018, 0.07231374084949493, 0.08016443997621536 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 6.081484684727911e-06, "first_to_last_ratio": 3.078323446728062, "imbalance_abs_log_ratio": 1.1243851133032405, "last_quartile_mean": 1.975583394653313e-06, "mean": 4.652151767814416e-06, "normalized": [ 2.4525720619095863, 1.654472314081446, 1.2359407881701103, 1.0825771642899618, 0.44240146398400065, 0.9876040497096084, 1.2882205604793382, 1.3141415846170936, 0.8999094901258098, 0.8300653322769579, 0.6617721713004259, 0.5902235973921949, 0.41188961598659773, 0.7734807523034054, 0.6403045835436688, 0.6070810068099217, 0.6510220287519627, 1.0769144150721777, 1.17728918889844, 1.3585829882761054, 1.8757747548401014, 2.9451631121793755, 2.0397595891560574, 1.6055563111889792, 0.7696742247332903, 0.787489387630775, 0.47642688311358905, 0.4043284660410758, 0.23119890902899107, 0.17833306500032328, 0.2510673329267495, 0.2987628061818795 ], "peak_layer": 22, "peak_normalized": 2.9451631121793755, "population_cv": 0.647782699862367, "rest_mean": 3.922293253119143e-06, "spike_contrast": 2.1909090403516185, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 8.59338774716889e-06, "top_five_layers": [ 22, 1, 23, 21, 2 ], "top_five_spike_overlap": 3 }, "values": [ 1.1409737453504931e-05, 7.696856300754007e-06, 5.749784122599522e-06, 5.0363132686470635e-06, 2.0581187527568545e-06, 4.594483925757231e-06, 5.992997557768831e-06, 6.113586096034851e-06, 4.186515525361756e-06, 3.86158990295371e-06, 3.078664576605661e-06, 2.7458097520138836e-06, 1.916173005156452e-06, 3.598349849198712e-06, 2.978794100272353e-06, 2.8242329790373333e-06, 3.028653281944571e-06, 5.00996929986286e-06, 5.476927981362678e-06, 6.3203342506312765e-06, 8.726388841751032e-06, 1.370134577882709e-05, 9.48927117860876e-06, 7.469291631423403e-06, 3.580641305234167e-06, 3.663520146801602e-06, 2.2164101665111957e-06, 1.880997388070682e-06, 1.0755724133559852e-06, 8.296324836010172e-07, 1.1680033367156284e-06, 1.3898899169362267e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 2.501881442640297e-05, "first_to_last_ratio": 3.237900394777418, "imbalance_abs_log_ratio": 1.1749250934411728, "last_quartile_mean": 7.726863515244986e-06, "mean": 1.8746275095793408e-05, "normalized": [ 2.7266943838272857, 1.6956801985242886, 1.2225736836170489, 1.084800406143391, 0.43326928120453706, 0.9708891520767708, 1.2340651886739802, 1.308842933942275, 0.8469867024287768, 0.8151560193806393, 0.6643331447918581, 0.5879794050531195, 0.3965947099676422, 0.7788494906893911, 0.6303148257793446, 0.6067295862798884, 0.6705457184875018, 1.122272629303958, 1.1400326955463402, 1.3293198974369922, 1.8727726070978334, 2.933889819892617, 2.0412526454455646, 1.5887046979826571, 0.7488650363562602, 0.7769080735385081, 0.4692747194609997, 0.3596226142236053, 0.2281363582819073, 0.17827317939681314, 0.24308882656814615, 0.29328136860005805 ], "peak_layer": 22, "peak_normalized": 2.933889819892617, "population_cv": 0.6701371859752334, "rest_mean": 1.584026585145761e-05, "spike_contrast": 2.174125443231605, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.443872501520673e-05, "top_five_layers": [ 22, 1, 23, 21, 2 ], "top_five_spike_overlap": 3 }, "values": [ 5.11153630213812e-05, 3.1787687476025894e-05, 2.2918702597962692e-05, 2.0335966837592423e-05, 8.122185136016924e-06, 1.8200555132352747e-05, 2.3134125513024628e-05, 2.4535929696867242e-05, 1.587784572620876e-05, 1.5281138985301368e-05, 1.2453771887521725e-05, 1.1022423677786719e-05, 7.43467353458982e-06, 1.4600526810681913e-05, 1.1816055121016689e-05, 1.137391973315971e-05, 1.2570234503073152e-05, 2.1038431441411376e-05, 2.1371366528910585e-05, 2.4919796487665735e-05, 3.510751048452221e-05, 5.499950566445477e-05, 3.82658836315386e-05, 2.9782295314362273e-05, 1.4038429981155787e-05, 1.4564132470695768e-05, 8.797152986517176e-06, 6.741584456904093e-06, 4.276706931705121e-06, 3.3419580631743884e-06, 4.557010015560081e-06, 5.497933216247475e-06 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.022186935355421156, "first_to_last_ratio": 3.237900405645793, "imbalance_abs_log_ratio": 1.1749250967977847, "last_quartile_mean": 0.006852259975858033, "mean": 0.016624384516035207, "normalized": [ 2.72669435852041, 1.6956801379542876, 1.2225737228522384, 1.084800440622922, 0.4332692907187762, 0.9708891703880574, 1.2340651843410115, 1.3088429879543189, 0.8469867498396527, 0.8151560188724276, 0.6643331498148527, 0.5879794060323777, 0.39659469075446624, 0.7788495080226716, 0.630314780171221, 0.6067295702945699, 0.6705457075269857, 1.1222726463658577, 1.1400326870437525, 1.3293198461802551, 1.8727726540228558, 2.9338897911629194, 2.041252540262143, 1.5887047747424334, 0.7488650208510843, 0.7769080837376301, 0.4692747421511952, 0.3596226167473466, 0.22813635877845012, 0.178273182701473, 0.24308881311021258, 0.2932813674611444 ], "peak_layer": 22, "peak_normalized": 2.9338897911629194, "population_cv": 0.6701371795288719, "rest_mean": 0.014047306427663123, "spike_contrast": 2.174125434688415, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.030540606193244457, "top_five_layers": [ 22, 1, 23, 21, 2 ], "top_five_spike_overlap": 3 }, "values": [ 0.04532961547374725, 0.028189638629555702, 0.02032453566789627, 0.018034139648079872, 0.007202835287898779, 0.01614043489098549, 0.020515574142336845, 0.021758709102869034, 0.014080633409321308, 0.013551467098295689, 0.011044129729270935, 0.009774795733392239, 0.006593142636120319, 0.01294789370149374, 0.010478595271706581, 0.01008650567382574, 0.011147409677505493, 0.01865709200501442, 0.018952341750264168, 0.022099124267697334, 0.03113369271159172, 0.0487741120159626, 0.033934567123651505, 0.026411239057779312, 0.012449420057237148, 0.01291561871767044, 0.007801403757184744, 0.005978504661470652, 0.0037926265504211187, 0.0029636819381266832, 0.004041201900690794, 0.004875622224062681 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 2.2502055010420463e-06, "first_to_last_ratio": 1.6203218878669938, "imbalance_abs_log_ratio": 0.4826248257210081, "last_quartile_mean": 1.388739804042416e-06, "mean": 1.540671790145609e-06, "normalized": [ 7.295441640710666, 1.9269521379257328, 0.3866790225286946, 0.49339548390158666, 0.2552067749878509, 0.38064517886390087, 0.41901258802805863, 0.5269493955734877, 0.449348240672511, 0.4829564252075093, 0.41193763515124854, 0.4041106839769221, 0.28061988632843293, 0.516105153614602, 0.4711465985442618, 0.601994855740353, 0.38422899569391034, 0.6791181438005177, 0.6214737220523961, 0.7974008865142206, 1.1492657674697155, 2.15764479041204, 2.0092792379708158, 1.6879996795881065, 1.1469178296982825, 1.5308746124867334, 1.2185391186358328, 1.0539092693883894, 0.5555289097150855, 0.5870930623748652, 0.5564409963558912, 0.5617832760873788 ], "peak_layer": 1, "peak_normalized": 7.295441640710666, "population_cv": 1.246460825113205, "rest_mean": 1.3608635630067443e-06, "spike_contrast": 1.8456194176778258, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.5116362166954787e-06, "top_five_layers": [ 1, 22, 23, 2, 24 ], "top_five_spike_overlap": 3 }, "values": [ 1.1239881132496521e-05, 2.9688007998629473e-06, 5.957454618510383e-07, 7.601605034324166e-07, 3.9318987887781987e-07, 5.864492891305417e-07, 6.455608740907337e-07, 8.118560685943521e-07, 6.922981583556975e-07, 7.440773401867773e-07, 6.346606937768229e-07, 6.226019308996911e-07, 4.323431426200841e-07, 7.951486509227834e-07, 7.258822734002024e-07, 9.274764920519374e-07, 5.919707746215863e-07, 1.0462981663295068e-06, 9.574870318829198e-07, 1.2285330512895598e-06, 1.770641347320634e-06, 3.3242224617424654e-06, 3.0956398404669017e-06, 2.6006534881162224e-06, 1.7670239458311698e-06, 2.358575329708401e-06, 1.8773688452711212e-06, 1.6237282807196607e-06, 8.558877198083792e-07, 9.045177193911513e-07, 8.572929459660372e-07, 8.655236456434068e-07 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 1.5287811606867763e-05, "first_to_last_ratio": 2.9552091250382673, "imbalance_abs_log_ratio": 1.0835694182484974, "last_quartile_mean": 5.17317420190011e-06, "mean": 1.2470142905840476e-05, "normalized": [ 1.4925090668316598, 1.7503641241928498, 1.275032194111587, 1.1190211425647878, 0.5081155198560083, 1.0814142228918597, 1.3529488136030112, 1.2282205204359686, 0.9151160070627982, 0.8585101285469784, 0.7105888922558973, 0.6274127988415052, 0.44536223507110867, 0.8478604833611738, 0.6877196292626275, 0.6274881394397667, 0.7187399718896789, 1.1692592453849575, 1.2024212156280287, 1.3841102101671754, 1.813721701840184, 3.048511951618062, 2.1998337346773074, 1.6169594822839144, 0.7668978573394593, 0.8533217650628342, 0.48253552693477897, 0.3197526465322443, 0.24104741227426527, 0.19258854733380207, 0.20819384253433884, 0.2544209701693806 ], "peak_layer": 22, "peak_normalized": 3.048511951618062, "population_cv": 0.6207277521130098, "rest_mean": 1.0416760805701077e-05, "spike_contrast": 2.2615865609297416, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.3558406246593223e-05, "top_five_layers": [ 22, 23, 21, 2, 24 ], "top_five_spike_overlap": 4 }, "values": [ 1.8611801351653412e-05, 2.1827290765941143e-05, 1.5899833670118824e-05, 1.3954353562439792e-05, 6.336273145279847e-06, 1.3485389899869915e-05, 1.687146504991688e-05, 1.531608540972229e-05, 1.1411627383495215e-05, 1.0705743989092298e-05, 8.86114503373392e-06, 7.823927262506913e-06, 5.553730716201244e-06, 1.0572941391728818e-05, 8.575962056056596e-06, 7.824866770533845e-06, 8.962790161604062e-06, 1.4580829883925617e-05, 1.4994364391895942e-05, 1.7260052118217573e-05, 2.2617368813371286e-05, 3.801537968683988e-05, 2.7432241040514782e-05, 2.0163715817034245e-05, 9.563325875205919e-06, 1.0641044354997575e-05, 6.01728697802173e-06, 3.987361196777783e-06, 3.0058956781431334e-06, 2.4016067072807346e-06, 2.5962069685192546e-06, 3.1726658562547527e-06 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 9.615166675303044e-06, "first_to_last_ratio": 2.773728591653114, "imbalance_abs_log_ratio": 1.0201924770386175, "last_quartile_mean": 3.466513163630225e-06, "mean": 8.275510602118175e-06, "normalized": [ 0.8690205521675091, 1.7847482215626658, 1.298451792341304, 1.1148598257760374, 0.5846759235992149, 1.145826017287574, 1.4271289447060713, 1.0703445929171338, 0.9536593911588405, 0.9345450252753307, 0.7711716643747089, 0.6555283879953705, 0.4943919712604837, 0.8933020678444843, 0.7291050468254533, 0.6414816556951427, 0.7413981837091306, 1.2031102074961837, 1.2545678362480475, 1.396370357966524, 1.7136225409340244, 3.043980076505096, 2.3425529000793155, 1.5850517736509329, 0.7538140261131501, 0.9219660883865931, 0.4984416930395485, 0.28915717973983046, 0.2554415163472741, 0.2092227209317715, 0.19597516809195364, 0.2270866499732998 ], "peak_layer": 22, "peak_normalized": 3.043980076505096, "population_cv": 0.6167401019384198, "rest_mean": 6.91494882529595e-06, "spike_contrast": 2.2592422000013235, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.5622544196958188e-05, "top_five_layers": [ 22, 23, 2, 21, 24 ], "top_five_spike_overlap": 4 }, "values": [ 7.191588792920811e-06, 1.4769702829653397e-05, 1.0745351573859807e-05, 9.226034308085218e-06, 4.838491804548539e-06, 9.482295354246162e-06, 1.1810220712504815e-05, 8.857648026605602e-06, 7.892018402344547e-06, 7.733837264822796e-06, 6.381839284586022e-06, 5.424832124845125e-06, 4.091345999768237e-06, 7.392530733341118e-06, 6.033716545061907e-06, 5.308588242769474e-06, 6.1354485296760686e-06, 9.956351277651265e-06, 1.0382189429947175e-05, 1.155567770183552e-05, 1.4181101505528204e-05, 2.5190489395754412e-05, 1.9385821360629052e-05, 1.3117112757754512e-05, 6.23819596512476e-06, 7.629740139236674e-06, 4.124859515286516e-06, 2.3929233066155575e-06, 2.1139089767530095e-06, 1.731424845274887e-06, 1.6217945812968537e-06, 1.8792579794535413e-06 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 4.588525507642771e-05, "first_to_last_ratio": 3.1014649090004522, "imbalance_abs_log_ratio": 1.1318745511643697, "last_quartile_mean": 1.4794703929510433e-05, "mean": 3.660603675825769e-05, "normalized": [ 1.7036912139018856, 1.7340168166565504, 1.2910943851692187, 1.130550320995967, 0.4373171909076479, 1.0189776002883344, 1.3200942328959056, 1.3921681993801769, 0.9041202072096598, 0.773996357394269, 0.6704210004689756, 0.6101613597879993, 0.39595760931218305, 0.8159815482776739, 0.6532996295853558, 0.6178547142277163, 0.7047021036635615, 1.1136466462434407, 1.168482517760107, 1.4256895276730468, 1.9364586156094123, 3.113466712142391, 2.1276195709921266, 1.7069501463514696, 0.772218911466579, 0.8014733260226635, 0.455270289214742, 0.3305776870543548, 0.22662433093498996, 0.17589991496151708, 0.20909015233416903, 0.2621271611159094 ], "peak_layer": 22, "peak_normalized": 3.113466712142391, "population_cv": 0.6471793909481345, "rest_mean": 3.0292561118754008e-05, "spike_contrast": 2.3338668835038914, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 7.069880521157757e-05, "top_five_layers": [ 22, 23, 21, 2, 24 ], "top_five_spike_overlap": 4 }, "values": [ 6.236538320081308e-05, 6.347548332996666e-05, 4.726184852188453e-05, 4.13849666074384e-05, 1.6008449165383354e-05, 3.7300731491995975e-05, 4.832341801375151e-05, 5.096176028018817e-05, 3.309625753900036e-05, 2.8332939109532163e-05, 2.4541455786675215e-05, 2.2335589164867997e-05, 1.449443880119361e-05, 2.986985055031255e-05, 2.3914710254757665e-05, 2.261721238028258e-05, 2.579635111032985e-05, 4.0766190068097785e-05, 4.277351399650797e-05, 5.2188843255862594e-05, 7.088607526384294e-05, 0.00011397167691029608, 7.788372022332624e-05, 6.248467980185524e-05, 2.8267873858567327e-05, 2.933876203314867e-05, 1.6665640941937454e-05, 1.2101138963771518e-05, 8.295818588521797e-06, 6.438998752855696e-06, 7.65396180213429e-06, 9.595436495146714e-06 ] } } }, "pre_mlp_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "e53fa425d89dfeb0e9395eca9bcd73dc0a55314abf45c5b10b43e2cbfa724345", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.059514161199331284, 0.08157797157764435, 0.14292451739311218, 0.10997853428125381, 0.0991024449467659, 0.1065012738108635, 0.12155740708112717, 0.1095810979604721, 0.06964563578367233, 0.08311934769153595, 0.09363401681184769, 0.0941167026758194, 0.08564268052577972, 0.09493856877088547, 0.09063401818275452, 0.09473779052495956, 0.10505300015211105, 0.09355910867452621, 0.09618847072124481, 0.11416846513748169, 0.047205954790115356, 0.053352247923612595, 0.06512638926506042, 0.07829347252845764, 0.05368548259139061, 0.05310077592730522, 0.061235878616571426, 0.07320034503936768, 0.05304349213838577, 0.0639304369688034, 0.07404901832342148, 0.08157160878181458 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 2.604846500275926e-06, "first_to_last_ratio": 0.3123796314435305, "imbalance_abs_log_ratio": 1.1635360633870153, "last_quartile_mean": 8.338720704159641e-06, "mean": 5.171276949766934e-06, "normalized": [ 1.2171609409189463, 0.4329698083277729, 0.29179320121064806, 0.4936899699243203, 0.2878361834023195, 0.23508515004618497, 0.31331526878498966, 0.757864421030719, 0.6806465468123495, 0.5140354604313097, 0.47557773124957586, 0.4852610183325604, 0.43577258510455763, 0.46648916310427724, 0.5583173988872333, 0.6010154134180351, 0.41567120812278135, 0.5483926803620757, 0.5253349883416192, 0.5384295333028697, 2.2151093431909157, 2.5117646113458765, 2.1935719525683863, 1.9048387662792023, 2.211162020453248, 2.2353081545336195, 1.9294683788039624, 1.2983555088827001, 1.59806333928576, 1.560360814338367, 1.0463761815790937, 1.020962257623723 ], "peak_layer": 22, "peak_normalized": 2.5117646113458765, "population_cv": 0.7113702243554939, "rest_mean": 4.015123703606217e-06, "spike_contrast": 2.842877410920807, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.1414504479034804e-05, "top_five_layers": [ 22, 26, 21, 25, 23 ], "top_five_spike_overlap": 4 }, "values": [ 6.29427631793078e-06, 2.2390067897504196e-06, 1.5089434555193293e-06, 2.5530075618007686e-06, 1.4884806205373025e-06, 1.2156904176663375e-06, 1.6202400274778483e-06, 3.91912681152462e-06, 3.5198117984691635e-06, 2.6582197278912645e-06, 2.4593441594333854e-06, 2.5094191187235992e-06, 2.2535007246915484e-06, 2.4123446564772166e-06, 2.8872138955193805e-06, 3.1080171538633294e-06, 2.1495509372471133e-06, 2.8358904273773078e-06, 2.716652716117096e-06, 2.7843682346428977e-06, 1.1454943887656555e-05, 1.2989030437893234e-05, 1.1343568075972144e-05, 9.850448805082124e-06, 1.1434531188569963e-05, 1.1559397535165772e-05, 9.977815352613106e-06, 6.714155915688025e-06, 8.264028110716026e-06, 8.06905791250756e-06, 5.411101028585108e-06, 5.279678589431569e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 1.0501670033136179e-05, "first_to_last_ratio": 0.3150383664120947, "imbalance_abs_log_ratio": 1.1550608494396604, "last_quartile_mean": 3.333457493681635e-05, "mean": 2.0682209211031477e-05, "normalized": [ 1.2671063530632423, 0.4403299430256648, 0.2936134454609503, 0.4983634028791479, 0.2878270517365504, 0.23438765460006605, 0.3112341663799096, 0.7292456333407806, 0.6783724479190344, 0.5148010714601226, 0.46707581755585115, 0.490777804795291, 0.43792703149518525, 0.4642070816408691, 0.5613443723948237, 0.5976948570890768, 0.4185039180483676, 0.5521873370995821, 0.5300093383307662, 0.5325107972459027, 2.212640057191234, 2.4974758593967135, 2.19262803407225, 1.8957271472420514, 2.2170100906127685, 2.232261584649462, 1.9383812345248908, 1.2992133636278134, 1.595579693560248, 1.550977173649396, 1.0479719734767008, 1.0126142624352872 ], "peak_layer": 22, "peak_normalized": 2.4974758593967135, "population_cv": 0.7106742818996169, "rest_mean": 1.607430400933178e-05, "spike_contrast": 2.8346420021518566, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.556489730020985e-05, "top_five_layers": [ 22, 26, 25, 21, 23 ], "top_five_spike_overlap": 4 }, "values": [ 2.6206558686681092e-05, 9.10699600353837e-06, 6.072574706195155e-06, 1.0307256161468104e-05, 5.952899300609715e-06, 4.847654508921551e-06, 6.437010142690269e-06, 1.5082410754985176e-05, 1.4030240890861023e-05, 1.064722346200142e-05, 9.660159776103683e-06, 1.0150369234906975e-05, 9.057298484549392e-06, 9.600827979738824e-06, 1.1609841749304906e-05, 1.2361650078673847e-05, 8.655585588712711e-06, 1.1420454029575922e-05, 1.0961764019157272e-05, 1.1013499715772923e-05, 4.576228457153775e-05, 5.1653318223543465e-05, 4.5348391722654924e-05, 3.920782546629198e-05, 4.5852666517021134e-05, 4.616810110746883e-05, 4.009000622318126e-05, 2.687060259631835e-05, 3.300011303508654e-05, 3.2077634386951104e-05, 2.1674375602742657e-05, 2.094310002576094e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.009312986047007143, "first_to_last_ratio": 0.3150383703376557, "imbalance_abs_log_ratio": 1.1550608369790798, "last_quartile_mean": 0.029561434173956513, "mean": 0.0183411899197381, "normalized": [ 1.267106361681887, 0.44032994322311025, 0.2936134500748175, 0.49836341800039896, 0.28782703186654907, 0.23438766523093857, 0.31123415883415156, 0.7292456479667809, 0.6783724315732925, 0.5148010582259618, 0.46707580242096014, 0.4907777952186674, 0.4379269923572508, 0.4642070681455, 0.5613443626055548, 0.5976948656177906, 0.4185039308099767, 0.5521873745787016, 0.5300093714760061, 0.5325107802107434, 2.2126400820762364, 2.4974759039686494, 2.1926279997558926, 1.8957272044355131, 2.217010006880578, 2.2322615826908514, 1.93838124527265, 1.2992133762453797, 1.5955797875471573, 1.550977103504217, 1.047971947923766, 1.0126142495800692 ], "peak_layer": 22, "peak_normalized": 2.4974759039686494, "population_cv": 0.7106742842829173, "rest_mean": 0.014254853508814617, "spike_contrast": 2.8346420055273533, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0404074065387249, "top_five_layers": [ 22, 26, 25, 21, 23 ], "top_five_spike_overlap": 4 }, "values": [ 0.023240238428115845, 0.00807617511600256, 0.005385220050811768, 0.009140578098595142, 0.005279090255498886, 0.004298948682844639, 0.005708404816687107, 0.013375232927501202, 0.012442157603800297, 0.009442063979804516, 0.008566725999116898, 0.00900144875049591, 0.008032102137804031, 0.008514109998941422, 0.01029572356492281, 0.01096243504434824, 0.0076758600771427155, 0.010127773508429527, 0.009721002541482449, 0.009766881354153156, 0.04058245196938515, 0.045806679874658585, 0.04021540656685829, 0.03476989269256592, 0.040662601590156555, 0.04094233363866806, 0.035552218556404114, 0.02382911927998066, 0.02926483191549778, 0.02844676561653614, 0.019221052527427673, 0.018572550266981125 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 6.276749928701975e-07, "first_to_last_ratio": 0.2775556275979974, "imbalance_abs_log_ratio": 1.2817339060724282, "last_quartile_mean": 2.2614385386532376e-06, "mean": 1.3446938584849022e-06, "normalized": [ 1.4224017295641362, 0.39187250748549846, 0.280493687844276, 0.5074145231368629, 0.2678947622854933, 0.20100321224564466, 0.23385179468970183, 0.42930060554071175, 0.5809259923520401, 0.4574558751084526, 0.4272286667760156, 0.44106665939933887, 0.4151291260582322, 0.44108542833906067, 0.5636837621504793, 0.5824793334759542, 0.4189417148925797, 0.5426125837008945, 0.5507932205316269, 0.4297354193109338, 2.1880535711880764, 2.512493632308928, 2.3056776709769595, 1.954406649620811, 2.231917259676148, 2.2763489199527376, 2.1220607954993436, 1.3555362827231123, 1.6235618146370503, 1.6219355113735872, 1.126390863077236, 1.0962464240780767 ], "peak_layer": 22, "peak_normalized": 2.512493632308928, "population_cv": 0.7551248145033022, "rest_mean": 1.0362834022662018e-06, "spike_contrast": 2.9047172959474294, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.010110322065884e-06, "top_five_layers": [ 22, 23, 26, 25, 21 ], "top_five_spike_overlap": 4 }, "values": [ 1.9126948700431967e-06, 5.269485541248287e-07, 3.7717813938797917e-07, 6.823171929681848e-07, 3.6023644156557566e-07, 2.7028778504245565e-07, 3.144590721149143e-07, 5.772778877144447e-07, 7.811676141500357e-07, 6.151381057861727e-07, 5.74491764382401e-07, 5.930996280767431e-07, 5.582215862887097e-07, 5.931248665547173e-07, 7.579820930914138e-07, 7.832563824194949e-07, 5.633483510791848e-07, 7.296478088392178e-07, 7.40648260943999e-07, 5.778625791208469e-07, 2.942262199212564e-06, 3.3785347568482393e-06, 3.1004306038084906e-06, 2.6280786187271588e-06, 3.0012454317329684e-06, 3.0609924124291865e-06, 2.8535221190395532e-06, 1.8227813143312233e-06, 2.183193601013045e-06, 2.181006721002632e-06, 1.5146508758334676e-06, 1.4741158338438254e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 7.015987165459592e-06, "first_to_last_ratio": 0.3262064008986602, "imbalance_abs_log_ratio": 1.1202249663676949, "last_quartile_mean": 2.1507815745280823e-05, "mean": 1.3879406132843997e-05, "normalized": [ 1.2734136404903196, 0.44795251444902995, 0.22566638103077008, 0.4515385538716205, 0.31518619520377167, 0.25483575647382595, 0.32943258458315156, 0.7459440356005865, 0.7424757506383413, 0.5591653442868995, 0.513485787643132, 0.5492440559222104, 0.48882969427848233, 0.5139409148575605, 0.6127939495233795, 0.6576017168847688, 0.4553956841136036, 0.6122947544878145, 0.5768508585089622, 0.5789761397623725, 2.2119716022066447, 2.4915066679975952, 2.184707349114303, 1.80982406567968, 2.2416150589418447, 2.1050205402249107, 1.8155418081348205, 1.2475617856385355, 1.5711897043989074, 1.47893704641395, 0.9973198619778846, 0.9397801966603206 ], "peak_layer": 22, "peak_normalized": 2.4915066679975952, "population_cv": 0.6820920997704788, "rest_mean": 1.082612968477608e-05, "spike_contrast": 2.804982005260251, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.036709895241074e-05, "top_five_layers": [ 22, 25, 21, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 1.7674225091468543e-05, 6.217314876266755e-06, 3.1321153528551804e-06, 6.267086973821279e-06, 4.374597210698994e-06, 3.536968961270759e-06, 4.572328634822043e-06, 1.0353260222473182e-05, 1.0305122486897744e-05, 7.760882908769418e-06, 7.126877790142316e-06, 7.623181318194838e-06, 6.784665856685024e-06, 7.13319468559348e-06, 8.505216101184487e-06, 9.127121302299201e-06, 6.320621650957037e-06, 8.498287570546381e-06, 8.006347343325615e-06, 8.035844984988216e-06, 3.0700852221343666e-05, 3.458063292782754e-05, 3.0322440579766408e-05, 2.5119283236563206e-05, 3.1112285796552896e-05, 2.921643499576021e-05, 2.5198642106261104e-05, 1.7315416698693298e-05, 2.180718001909554e-05, 2.0526767912087962e-05, 1.384220740874298e-05, 1.3043591025052592e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 4.570314985130608e-06, "first_to_last_ratio": 0.32482178840378695, "imbalance_abs_log_ratio": 1.1244785904205714, "last_quartile_mean": 1.4070222960071987e-05, "mean": 9.271555178713697e-06, "normalized": [ 1.227028819819665, 0.43710546686488033, 0.18221482779192477, 0.3896796444161369, 0.33205074803886947, 0.2665486849592955, 0.3382090641769959, 0.7706780632378566, 0.7763987768609354, 0.5672032427801852, 0.5320079250559773, 0.588342357683285, 0.5252786460308668, 0.5609642924193802, 0.6473950072332498, 0.7154699592538526, 0.4901338473147847, 0.6755094728738558, 0.6264360931416593, 0.6178016589392343, 2.244387864500746, 2.468773090992691, 2.17445935535773, 1.705372609684643, 2.2438359811226563, 1.9834565008644034, 1.7168075626501473, 1.2669392286197625, 1.5492694631466168, 1.4990842728550053, 0.9949037869973498, 0.8862536843153582 ], "peak_layer": 22, "peak_normalized": 2.468773090992691, "population_cv": 0.6621513870135299, "rest_mean": 7.267241059067871e-06, "spike_contrast": 2.7651279572909364, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.0094851424801164e-05, "top_five_layers": [ 22, 21, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 1.1376465408829972e-05, 4.0526474549551494e-06, 1.6894148302526446e-06, 3.6129363252257463e-06, 3.0786268325755373e-06, 2.471320840413682e-06, 3.13572400045814e-06, 7.145384188333992e-06, 7.1984241003519855e-06, 5.258856162981829e-06, 4.932540832669474e-06, 5.454848633235088e-06, 4.870149950875202e-06, 5.201011390454369e-06, 6.002358531986829e-06, 6.633519205934135e-06, 4.544303010334261e-06, 6.263023351493757e-06, 5.808036803500727e-06, 5.727982170355972e-06, 2.0808965928154066e-05, 2.2889365936862305e-05, 2.016061989706941e-05, 1.5811456250958145e-05, 2.0803849110961892e-05, 1.838972639234271e-05, 1.5917476048343815e-05, 1.1746496966225095e-05, 1.4364137314260006e-05, 1.3898842553317081e-05, 9.224305358657148e-06, 8.216949936468154e-06 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 2.112927961661626e-05, "first_to_last_ratio": 0.2924447431524573, "imbalance_abs_log_ratio": 1.2294795424556932, "last_quartile_mean": 7.225050239867414e-05, "mean": 4.3442557995376774e-05, "normalized": [ 1.2473462685168368, 0.4335038817899225, 0.2427926911636276, 0.46141836554862276, 0.282706103609971, 0.23202630227217172, 0.31095827810827176, 0.6802307014732507, 0.6873979100249731, 0.517630035280524, 0.47005596290413315, 0.49025288385428417, 0.43635673000467046, 0.45557725785417114, 0.5645871782883749, 0.5960677432510797, 0.4230061590646969, 0.5530453987011605, 0.5252778826419021, 0.5248241669749075, 2.0588893434924276, 2.402854222111511, 2.175740639694511, 1.9224358221336895, 2.2264864508836406, 2.2526971142361627, 2.038805265263073, 1.3703716317142152, 1.6701662266244346, 1.6144111197970101, 1.0759419021584327, 1.05613836056334 ], "peak_layer": 22, "peak_normalized": 2.402854222111511, "population_cv": 0.7114350003749988, "rest_mean": 3.413232469837466e-05, "spike_contrast": 2.7457203289657834, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.37178177991882e-05, "top_five_layers": [ 22, 26, 25, 23, 21 ], "top_five_spike_overlap": 4 }, "values": [ 5.418791261035949e-05, 1.8832517525879666e-05, 1.0547535566729493e-05, 2.0045194105478004e-05, 1.228147630172316e-05, 1.0079816092911642e-05, 1.3508823030861095e-05, 2.955096169898752e-05, 2.9862323572160676e-05, 2.2487172827823088e-05, 2.042043342953548e-05, 2.1297839339240454e-05, 1.895645254990086e-05, 1.979144144570455e-05, 2.4527111236238852e-05, 2.5894707505358383e-05, 1.8376469597569667e-05, 2.4025706807151437e-05, 2.2819414880359545e-05, 2.2799704311182722e-05, 8.9443419710733e-05, 0.00010438613389851525, 9.451973892282695e-05, 8.351552969543263e-05, 9.672426676843315e-05, 9.78629250312224e-05, 8.857091597747058e-05, 5.953244908596389e-05, 7.255629316205159e-05, 7.013414870016277e-05, 4.6741668484173715e-05, 4.588135197991505e-05 ] } } } }, "selector": { "expected_uniform_count": 49, "identities_unique": true, "identity_order_sha256": "97fd9721a263024f6a746638f32e0ad9a988cd4c111c00e9e178c1022e5c9d5e", "passed": true, "uniform_count": 49, "uniform_indices": [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 56, 57, 58, 59, 60, 61, 62, 63, 64 ], "visit_count": 65, "visits": [ { "branch": "attention", "coefficient": "uniform", "group": 1, "index": 0, "kind": "depth", "layer": 1, "offset": 1, "sources": 1, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 1, "index": 1, "kind": "depth", "layer": 1, "offset": 1, "sources": 2, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 1, "index": 2, "kind": "depth", "layer": 2, "offset": 2, "sources": 2, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 1, "index": 3, "kind": "depth", "layer": 2, "offset": 2, "sources": 2, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 1, "index": 4, "kind": "depth", "layer": 3, "offset": 3, "sources": 2, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 1, "index": 5, "kind": "depth", "layer": 3, "offset": 3, "sources": 2, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 1, "index": 6, "kind": "depth", "layer": 4, "offset": 4, "sources": 2, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 1, "index": 7, "kind": "depth", "layer": 4, "offset": 4, "sources": 2, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 2, "index": 8, "kind": "depth", "layer": 5, "offset": 1, "sources": 2, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 2, "index": 9, "kind": "depth", "layer": 5, "offset": 1, "sources": 3, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 2, "index": 10, "kind": "depth", "layer": 6, "offset": 2, "sources": 3, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 2, "index": 11, "kind": "depth", "layer": 6, "offset": 2, "sources": 3, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 2, "index": 12, "kind": "depth", "layer": 7, "offset": 3, "sources": 3, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 2, "index": 13, "kind": "depth", "layer": 7, "offset": 3, "sources": 3, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 2, "index": 14, "kind": "depth", "layer": 8, "offset": 4, "sources": 3, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 2, "index": 15, "kind": "depth", "layer": 8, "offset": 4, "sources": 3, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 3, "index": 16, "kind": "depth", "layer": 9, "offset": 1, "sources": 3, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 3, "index": 17, "kind": "depth", "layer": 9, "offset": 1, "sources": 4, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 3, "index": 18, "kind": "depth", "layer": 10, "offset": 2, "sources": 4, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 3, "index": 19, "kind": "depth", "layer": 10, "offset": 2, "sources": 4, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 3, "index": 20, "kind": "depth", "layer": 11, "offset": 3, "sources": 4, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 3, "index": 21, "kind": "depth", "layer": 11, "offset": 3, "sources": 4, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 3, "index": 22, "kind": "depth", "layer": 12, "offset": 4, "sources": 4, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 3, "index": 23, "kind": "depth", "layer": 12, "offset": 4, "sources": 4, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 4, "index": 24, "kind": "depth", "layer": 13, "offset": 1, "sources": 4, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 4, "index": 25, "kind": "depth", "layer": 13, "offset": 1, "sources": 5, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 4, "index": 26, "kind": "depth", "layer": 14, "offset": 2, "sources": 5, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 4, "index": 27, "kind": "depth", "layer": 14, "offset": 2, "sources": 5, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 4, "index": 28, "kind": "depth", "layer": 15, "offset": 3, "sources": 5, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 4, "index": 29, "kind": "depth", "layer": 15, "offset": 3, "sources": 5, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 4, "index": 30, "kind": "depth", "layer": 16, "offset": 4, "sources": 5, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 4, "index": 31, "kind": "depth", "layer": 16, "offset": 4, "sources": 5, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 5, "index": 32, "kind": "depth", "layer": 17, "offset": 1, "sources": 5, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 5, "index": 33, "kind": "depth", "layer": 17, "offset": 1, "sources": 6, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 5, "index": 34, "kind": "depth", "layer": 18, "offset": 2, "sources": 6, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 5, "index": 35, "kind": "depth", "layer": 18, "offset": 2, "sources": 6, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 5, "index": 36, "kind": "depth", "layer": 19, "offset": 3, "sources": 6, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 5, "index": 37, "kind": "depth", "layer": 19, "offset": 3, "sources": 6, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 5, "index": 38, "kind": "depth", "layer": 20, "offset": 4, "sources": 6, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 5, "index": 39, "kind": "depth", "layer": 20, "offset": 4, "sources": 6, "use_uniform": true }, { "branch": "attention", "coefficient": "detached_learned", "group": 6, "index": 40, "kind": "depth", "layer": 21, "offset": 1, "sources": 6, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 6, "index": 41, "kind": "depth", "layer": 21, "offset": 1, "sources": 7, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 6, "index": 42, "kind": "depth", "layer": 22, "offset": 2, "sources": 7, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 6, "index": 43, "kind": "depth", "layer": 22, "offset": 2, "sources": 7, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 6, "index": 44, "kind": "depth", "layer": 23, "offset": 3, "sources": 7, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 6, "index": 45, "kind": "depth", "layer": 23, "offset": 3, "sources": 7, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 6, "index": 46, "kind": "depth", "layer": 24, "offset": 4, "sources": 7, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 6, "index": 47, "kind": "depth", "layer": 24, "offset": 4, "sources": 7, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 7, "index": 48, "kind": "depth", "layer": 25, "offset": 1, "sources": 7, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 7, "index": 49, "kind": "depth", "layer": 25, "offset": 1, "sources": 8, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 7, "index": 50, "kind": "depth", "layer": 26, "offset": 2, "sources": 8, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 7, "index": 51, "kind": "depth", "layer": 26, "offset": 2, "sources": 8, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 7, "index": 52, "kind": "depth", "layer": 27, "offset": 3, "sources": 8, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 7, "index": 53, "kind": "depth", "layer": 27, "offset": 3, "sources": 8, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 7, "index": 54, "kind": "depth", "layer": 28, "offset": 4, "sources": 8, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 7, "index": 55, "kind": "depth", "layer": 28, "offset": 4, "sources": 8, "use_uniform": false }, { "branch": "attention", "coefficient": "uniform", "group": 8, "index": 56, "kind": "depth", "layer": 29, "offset": 1, "sources": 8, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 8, "index": 57, "kind": "depth", "layer": 29, "offset": 1, "sources": 9, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 8, "index": 58, "kind": "depth", "layer": 30, "offset": 2, "sources": 9, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 8, "index": 59, "kind": "depth", "layer": 30, "offset": 2, "sources": 9, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 8, "index": 60, "kind": "depth", "layer": 31, "offset": 3, "sources": 9, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 8, "index": 61, "kind": "depth", "layer": 31, "offset": 3, "sources": 9, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 8, "index": 62, "kind": "depth", "layer": 32, "offset": 4, "sources": 9, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 8, "index": 63, "kind": "depth", "layer": 32, "offset": 4, "sources": 9, "use_uniform": true }, { "branch": "output", "coefficient": "uniform", "group": null, "index": 64, "kind": "output", "layer": null, "offset": null, "sources": 9, "use_uniform": true } ], "visits_sha256": "dd2c48ef3a1efa7d1500720ce6e1f326e1c0b8cb70134a5b5e65d9d7680b9574" } }, "uniform_group_6_attention_only": { "bits_per_byte": 1.8740129107275643, "forward": { "activation_sha256": { "attention_branch_output": "5e6ecbcd78543fc42d4098a51169dca446d404ce384015643e84d2c78433c949", "mlp_branch_output": "c0b8018c349b1bd09e5d17858eb312a2323da0e92de9651e2bb08ae4adf115c7", "post_attention_state": "90a633f12d984935028fe5d7212be48e7da7b4a0806751939a551938d16dc540", "post_mlp_state": "89f2226456bfe96e245d2aad0f699d32c676f9f53099789cd870253d002402f5", "pre_attention_input": "ee695e78937de01b7128d3866b8fc5dd328ebdf2c159be37918d4f02952d5248", "pre_mlp_input": "e53fa425d89dfeb0e9395eca9bcd73dc0a55314abf45c5b10b43e2cbfa724345" }, "logits_sha256": "bdacba5675f0579cec629bbf01b36c8660ad576cfdf9fef1bea8259eaeac9ec2", "loss_nats": 1.2989667654037476, "mixer_summary_sha256": "eb3ac0f07de9b755d6e67a510ab68d09d9ad666bb7b5d020ba7d8c5055c399b2" }, "loss_nats": 1.2989667654037476, "loss_scale": 1.0, "mixers": [ { "branch": "attention", "entropy_mean": -0.0, "group": 1, "latest_source_mass": 1.0, "layer": 1, "max_source_mass": 1.0, "mixer_index": 0, "normalized_entropy": 1.0, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 1.0, "median": 1.0, "p05": 1.0, "p95": 1.0 } ], "sources": 1 }, { "branch": "mlp", "entropy_mean": 0.22886337339878082, "group": 1, "latest_source_mass": 0.06350618600845337, "layer": 1, "max_source_mass": 0.9364938139915466, "mixer_index": 1, "normalized_entropy": 0.3301800538435402, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.9364938139915466, "median": 0.9452826976776123, "p05": 0.8725647330284119, "p95": 0.9712384939193726 }, { "label": "current_group_partial", "mean": 0.06350618600845337, "median": 0.0547172985970974, "p05": 0.028761589899659157, "p95": 0.12743529677391052 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.603729784488678, "group": 1, "latest_source_mass": 0.4133155345916748, "layer": 2, "max_source_mass": 0.5866844654083252, "mixer_index": 2, "normalized_entropy": 0.8709979661187783, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.5866844654083252, "median": 0.5734395384788513, "p05": 0.29813364148139954, "p95": 0.8843642473220825 }, { "label": "current_group_partial", "mean": 0.4133155345916748, "median": 0.4265604615211487, "p05": 0.11563576757907867, "p95": 0.7018663287162781 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.3967982828617096, "group": 1, "latest_source_mass": 0.14110636711120605, "layer": 2, "max_source_mass": 0.858893632888794, "mixer_index": 3, "normalized_entropy": 0.5724589149178446, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.858893632888794, "median": 0.8705973029136658, "p05": 0.7599387168884277, "p95": 0.9270997643470764 }, { "label": "current_group_partial", "mean": 0.14110636711120605, "median": 0.12940272688865662, "p05": 0.07290027290582657, "p95": 0.2400612235069275 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6852496862411499, "group": 1, "latest_source_mass": 0.518975019454956, "layer": 3, "max_source_mass": 0.518975019454956, "mixer_index": 4, "normalized_entropy": 0.9886063241108252, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.48102498054504395, "median": 0.48102718591690063, "p05": 0.38786420226097107, "p95": 0.580143928527832 }, { "label": "current_group_partial", "mean": 0.518975019454956, "median": 0.5189728140830994, "p05": 0.41985607147216797, "p95": 0.6121357679367065 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.5890734195709229, "group": 1, "latest_source_mass": 0.3123423159122467, "layer": 3, "max_source_mass": 0.6876576542854309, "mixer_index": 5, "normalized_entropy": 0.8498533011344741, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.6876576542854309, "median": 0.6909762620925903, "p05": 0.48803940415382385, "p95": 0.8900076746940613 }, { "label": "current_group_partial", "mean": 0.3123423159122467, "median": 0.3090237081050873, "p05": 0.10999235510826111, "p95": 0.5119606256484985 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6873323321342468, "group": 1, "latest_source_mass": 0.4831485152244568, "layer": 4, "max_source_mass": 0.5168514251708984, "mixer_index": 6, "normalized_entropy": 0.9916109470127238, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.5168514251708984, "median": 0.5219584703445435, "p05": 0.4251725673675537, "p95": 0.5947866439819336 }, { "label": "current_group_partial", "mean": 0.4831485152244568, "median": 0.47804147005081177, "p05": 0.405213326215744, "p95": 0.5748274326324463 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.487301766872406, "group": 1, "latest_source_mass": 0.2060384750366211, "layer": 4, "max_source_mass": 0.7939615249633789, "mixer_index": 7, "normalized_entropy": 0.7030278424832499, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.7939615249633789, "median": 0.7953404188156128, "p05": 0.6590759754180908, "p95": 0.9329437017440796 }, { "label": "current_group_partial", "mean": 0.2060384750366211, "median": 0.20465964078903198, "p05": 0.06705628335475922, "p95": 0.3409239649772644 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.674697756767273, "group": 2, "latest_source_mass": 0.42952626943588257, "layer": 5, "max_source_mass": 0.5704737305641174, "mixer_index": 8, "normalized_entropy": 0.9733831077870527, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.5704737305641174, "median": 0.5696132779121399, "p05": 0.4695202708244324, "p95": 0.6750621199607849 }, { "label": "completed_group_1", "mean": 0.42952626943588257, "median": 0.4303867816925049, "p05": 0.3249378800392151, "p95": 0.5304797887802124 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.9691112041473389, "group": 2, "latest_source_mass": 0.2681158185005188, "layer": 5, "max_source_mass": 0.5506307482719421, "mixer_index": 9, "normalized_entropy": 0.8821230329784768, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.5506307482719421, "median": 0.543993353843689, "p05": 0.41262108087539673, "p95": 0.7161091566085815 }, { "label": "completed_group_1", "mean": 0.18125343322753906, "median": 0.1794438362121582, "p05": 0.11575750261545181, "p95": 0.25031501054763794 }, { "label": "current_group_partial", "mean": 0.2681158185005188, "median": 0.2664111256599426, "p05": 0.1564716249704361, "p95": 0.3817881643772125 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0745112895965576, "group": 2, "latest_source_mass": 0.412794291973114, "layer": 6, "max_source_mass": 0.412794291973114, "mixer_index": 10, "normalized_entropy": 0.9780623252441762, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.34262996912002563, "median": 0.3412924110889435, "p05": 0.29556700587272644, "p95": 0.39125555753707886 }, { "label": "completed_group_1", "mean": 0.24457570910453796, "median": 0.24382859468460083, "p05": 0.22120657563209534, "p95": 0.2734297513961792 }, { "label": "current_group_partial", "mean": 0.412794291973114, "median": 0.4151499271392822, "p05": 0.37735897302627563, "p95": 0.43749260902404785 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0074045658111572, "group": 2, "latest_source_mass": 0.3775783181190491, "layer": 6, "max_source_mass": 0.4399106502532959, "mixer_index": 11, "normalized_entropy": 0.9169791528842925, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.4399106502532959, "median": 0.44856905937194824, "p05": 0.22063353657722473, "p95": 0.6359403133392334 }, { "label": "completed_group_1", "mean": 0.18251106142997742, "median": 0.18562838435173035, "p05": 0.1196163073182106, "p95": 0.23045453429222107 }, { "label": "current_group_partial", "mean": 0.3775783181190491, "median": 0.3675365447998047, "p05": 0.23817461729049683, "p95": 0.5557451844215393 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.060497522354126, "group": 2, "latest_source_mass": 0.4025024175643921, "layer": 7, "max_source_mass": 0.4025024175643921, "mixer_index": 12, "normalized_entropy": 0.9653064445872968, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.3786153197288513, "median": 0.3907395303249359, "p05": 0.28039106726646423, "p95": 0.4436405599117279 }, { "label": "completed_group_1", "mean": 0.2188822478055954, "median": 0.21226292848587036, "p05": 0.1885872334241867, "p95": 0.2672030031681061 }, { "label": "current_group_partial", "mean": 0.4025024175643921, "median": 0.39815032482147217, "p05": 0.3533596992492676, "p95": 0.460100919008255 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 0.9941019415855408, "group": 2, "latest_source_mass": 0.4801652431488037, "layer": 7, "max_source_mass": 0.4801652431488037, "mixer_index": 13, "normalized_entropy": 0.9048705824970601, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.32296955585479736, "median": 0.2803739905357361, "p05": 0.1784922033548355, "p95": 0.6440245509147644 }, { "label": "completed_group_1", "mean": 0.19686520099639893, "median": 0.2053825855255127, "p05": 0.09813719242811203, "p95": 0.2504696547985077 }, { "label": "current_group_partial", "mean": 0.4801652431488037, "median": 0.5057140588760376, "p05": 0.2529003322124481, "p95": 0.598870575428009 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0503522157669067, "group": 2, "latest_source_mass": 0.4001518487930298, "layer": 8, "max_source_mass": 0.40130966901779175, "mixer_index": 14, "normalized_entropy": 0.9560717885654542, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.40130966901779175, "median": 0.396759569644928, "p05": 0.3479611873626709, "p95": 0.4733521044254303 }, { "label": "completed_group_1", "mean": 0.19853851199150085, "median": 0.19769755005836487, "p05": 0.17136667668819427, "p95": 0.22788912057876587 }, { "label": "current_group_partial", "mean": 0.4001518487930298, "median": 0.40291696786880493, "p05": 0.3506356179714203, "p95": 0.43941178917884827 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0129402875900269, "group": 2, "latest_source_mass": 0.42161279916763306, "layer": 8, "max_source_mass": 0.42161279916763306, "mixer_index": 15, "normalized_entropy": 0.9220179839951123, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.4093267023563385, "median": 0.4046899676322937, "p05": 0.2895161807537079, "p95": 0.5762131810188293 }, { "label": "completed_group_1", "mean": 0.16906049847602844, "median": 0.16888955235481262, "p05": 0.11233551055192947, "p95": 0.22477403283119202 }, { "label": "current_group_partial", "mean": 0.42161279916763306, "median": 0.4227708578109741, "p05": 0.3131049573421478, "p95": 0.519075334072113 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0774019956588745, "group": 3, "latest_source_mass": 0.3560434877872467, "layer": 9, "max_source_mass": 0.38510820269584656, "mixer_index": 16, "normalized_entropy": 0.9806935592947451, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.38510820269584656, "median": 0.3892287313938141, "p05": 0.2934730052947998, "p95": 0.4718902111053467 }, { "label": "completed_group_1", "mean": 0.25884830951690674, "median": 0.25734007358551025, "p05": 0.22217319905757904, "p95": 0.29551801085472107 }, { "label": "completed_group_2", "mean": 0.3560434877872467, "median": 0.35412120819091797, "p05": 0.29444968700408936, "p95": 0.41908568143844604 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.2740377187728882, "group": 3, "latest_source_mass": 0.3633861243724823, "layer": 9, "max_source_mass": 0.3633861243724823, "mixer_index": 17, "normalized_entropy": 0.9190239493895668, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.29550763964653015, "median": 0.30360478162765503, "p05": 0.16904757916927338, "p95": 0.45146965980529785 }, { "label": "completed_group_1", "mean": 0.09541244804859161, "median": 0.09486871212720871, "p05": 0.061360105872154236, "p95": 0.13037073612213135 }, { "label": "completed_group_2", "mean": 0.24569380283355713, "median": 0.24333015084266663, "p05": 0.1698218435049057, "p95": 0.31589236855506897 }, { "label": "current_group_partial", "mean": 0.3633861243724823, "median": 0.36472851037979126, "p05": 0.2795379161834717, "p95": 0.4399876892566681 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.360302448272705, "group": 3, "latest_source_mass": 0.28569501638412476, "layer": 10, "max_source_mass": 0.3109527826309204, "mixer_index": 18, "normalized_entropy": 0.9812507981160736, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.3109527826309204, "median": 0.32172977924346924, "p05": 0.23520725965499878, "p95": 0.37052470445632935 }, { "label": "completed_group_1", "mean": 0.20984819531440735, "median": 0.20958667993545532, "p05": 0.1891367882490158, "p95": 0.23184512555599213 }, { "label": "completed_group_2", "mean": 0.19350400567054749, "median": 0.19153150916099548, "p05": 0.16607457399368286, "p95": 0.22877082228660583 }, { "label": "current_group_partial", "mean": 0.28569501638412476, "median": 0.28060320019721985, "p05": 0.23824527859687805, "p95": 0.3439165949821472 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.2722704410552979, "group": 3, "latest_source_mass": 0.29912811517715454, "layer": 10, "max_source_mass": 0.35036465525627136, "mixer_index": 19, "normalized_entropy": 0.9177491279900463, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.35036465525627136, "median": 0.36210548877716064, "p05": 0.12807878851890564, "p95": 0.5797606706619263 }, { "label": "completed_group_1", "mean": 0.12631990015506744, "median": 0.1264108121395111, "p05": 0.06977735459804535, "p95": 0.1755298376083374 }, { "label": "completed_group_2", "mean": 0.22418735921382904, "median": 0.22563719749450684, "p05": 0.1176326721906662, "p95": 0.32001155614852905 }, { "label": "current_group_partial", "mean": 0.29912811517715454, "median": 0.28589946031570435, "p05": 0.193109393119812, "p95": 0.44119665026664734 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3558036088943481, "group": 3, "latest_source_mass": 0.314197838306427, "layer": 11, "max_source_mass": 0.314197838306427, "mixer_index": 20, "normalized_entropy": 0.978005571485618, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.298976868391037, "median": 0.2961939573287964, "p05": 0.24950163066387177, "p95": 0.35835936665534973 }, { "label": "completed_group_1", "mean": 0.1941717267036438, "median": 0.1918032318353653, "p05": 0.16809208691120148, "p95": 0.22895419597625732 }, { "label": "completed_group_2", "mean": 0.1926535815000534, "median": 0.19306811690330505, "p05": 0.16983944177627563, "p95": 0.2155935913324356 }, { "label": "current_group_partial", "mean": 0.314197838306427, "median": 0.31541913747787476, "p05": 0.2673022747039795, "p95": 0.35941022634506226 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.2696614265441895, "group": 3, "latest_source_mass": 0.2862822115421295, "layer": 11, "max_source_mass": 0.3667730689048767, "mixer_index": 21, "normalized_entropy": 0.9158671218416545, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.3667730689048767, "median": 0.4067659378051758, "p05": 0.10593366622924805, "p95": 0.5672274827957153 }, { "label": "completed_group_1", "mean": 0.15310388803482056, "median": 0.15050218999385834, "p05": 0.08651046454906464, "p95": 0.22459912300109863 }, { "label": "completed_group_2", "mean": 0.19384080171585083, "median": 0.18732509016990662, "p05": 0.1267009675502777, "p95": 0.281107634305954 }, { "label": "current_group_partial", "mean": 0.2862822115421295, "median": 0.26271265745162964, "p05": 0.17024393379688263, "p95": 0.475044310092926 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3606936931610107, "group": 3, "latest_source_mass": 0.3028184771537781, "layer": 12, "max_source_mass": 0.3028184771537781, "mixer_index": 22, "normalized_entropy": 0.9815330216461395, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.298227459192276, "median": 0.2954302132129669, "p05": 0.25287362933158875, "p95": 0.35405808687210083 }, { "label": "completed_group_1", "mean": 0.18611131608486176, "median": 0.1850002110004425, "p05": 0.15901757776737213, "p95": 0.21725836396217346 }, { "label": "completed_group_2", "mean": 0.21284276247024536, "median": 0.21402311325073242, "p05": 0.18819917738437653, "p95": 0.23418807983398438 }, { "label": "current_group_partial", "mean": 0.3028184771537781, "median": 0.3041122555732727, "p05": 0.25661808252334595, "p95": 0.34458184242248535 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.269316554069519, "group": 3, "latest_source_mass": 0.2653314769268036, "layer": 12, "max_source_mass": 0.4081852436065674, "mixer_index": 23, "normalized_entropy": 0.9156183489371815, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.4081852436065674, "median": 0.4254342317581177, "p05": 0.20765496790409088, "p95": 0.587238073348999 }, { "label": "completed_group_1", "mean": 0.14420127868652344, "median": 0.1438048928976059, "p05": 0.1064920499920845, "p95": 0.1871720254421234 }, { "label": "completed_group_2", "mean": 0.18228203058242798, "median": 0.17507661879062653, "p05": 0.12183455377817154, "p95": 0.26505547761917114 }, { "label": "current_group_partial", "mean": 0.2653314769268036, "median": 0.24924281239509583, "p05": 0.15609969198703766, "p95": 0.4061311185359955 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3636136054992676, "group": 4, "latest_source_mass": 0.3136237859725952, "layer": 13, "max_source_mass": 0.3136237859725952, "mixer_index": 24, "normalized_entropy": 0.9836392931712563, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.27427661418914795, "median": 0.27403637766838074, "p05": 0.22928935289382935, "p95": 0.319773405790329 }, { "label": "completed_group_1", "mean": 0.21341991424560547, "median": 0.20697325468063354, "p05": 0.18552586436271667, "p95": 0.26688331365585327 }, { "label": "completed_group_2", "mean": 0.19867971539497375, "median": 0.197545126080513, "p05": 0.1794913411140442, "p95": 0.222100168466568 }, { "label": "completed_group_3", "mean": 0.3136237859725952, "median": 0.31875866651535034, "p05": 0.24156072735786438, "p95": 0.3695686459541321 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.570773959159851, "group": 4, "latest_source_mass": 0.2294243425130844, "layer": 13, "max_source_mass": 0.2658740282058716, "mixer_index": 25, "normalized_entropy": 0.9759767351225285, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.2658740282058716, "median": 0.2747163474559784, "p05": 0.17927880585193634, "p95": 0.3512483835220337 }, { "label": "completed_group_1", "mean": 0.13419361412525177, "median": 0.13374894857406616, "p05": 0.11072897166013718, "p95": 0.1586270034313202 }, { "label": "completed_group_2", "mean": 0.1907704919576645, "median": 0.19074805080890656, "p05": 0.14699944853782654, "p95": 0.23316305875778198 }, { "label": "completed_group_3", "mean": 0.17973749339580536, "median": 0.17616114020347595, "p05": 0.1363743096590042, "p95": 0.23202598094940186 }, { "label": "current_group_partial", "mean": 0.2294243425130844, "median": 0.22995705902576447, "p05": 0.17975632846355438, "p95": 0.2789157032966614 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.5560038089752197, "group": 4, "latest_source_mass": 0.32468360662460327, "layer": 14, "max_source_mass": 0.32468360662460327, "mixer_index": 26, "normalized_entropy": 0.9667995248241249, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.20477107167243958, "median": 0.206403911113739, "p05": 0.16619175672531128, "p95": 0.2431035190820694 }, { "label": "completed_group_1", "mean": 0.15721669793128967, "median": 0.15658551454544067, "p05": 0.13878680765628815, "p95": 0.17778633534908295 }, { "label": "completed_group_2", "mean": 0.13811248540878296, "median": 0.13743364810943604, "p05": 0.12352626770734787, "p95": 0.15553249418735504 }, { "label": "completed_group_3", "mean": 0.1752161681652069, "median": 0.17309701442718506, "p05": 0.15223677456378937, "p95": 0.20474471151828766 }, { "label": "current_group_partial", "mean": 0.32468360662460327, "median": 0.32497870922088623, "p05": 0.2946181893348694, "p95": 0.35203880071640015 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.5659973621368408, "group": 4, "latest_source_mass": 0.26703086495399475, "layer": 14, "max_source_mass": 0.26703086495399475, "mixer_index": 27, "normalized_entropy": 0.9730088685238187, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.20531430840492249, "median": 0.20229148864746094, "p05": 0.10698921978473663, "p95": 0.3393132984638214 }, { "label": "completed_group_1", "mean": 0.15986156463623047, "median": 0.15952220559120178, "p05": 0.11969029903411865, "p95": 0.20180393755435944 }, { "label": "completed_group_2", "mean": 0.21557100117206573, "median": 0.21654027700424194, "p05": 0.1673085242509842, "p95": 0.2610440254211426 }, { "label": "completed_group_3", "mean": 0.15222224593162537, "median": 0.1510905921459198, "p05": 0.11800260841846466, "p95": 0.18893754482269287 }, { "label": "current_group_partial", "mean": 0.26703086495399475, "median": 0.258539617061615, "p05": 0.19330690801143646, "p95": 0.36521556973457336 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.5552496910095215, "group": 4, "latest_source_mass": 0.31774693727493286, "layer": 15, "max_source_mass": 0.31774693727493286, "mixer_index": 28, "normalized_entropy": 0.9663309649872576, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.2162160575389862, "median": 0.2145705670118332, "p05": 0.17727568745613098, "p95": 0.26878347992897034 }, { "label": "completed_group_1", "mean": 0.17982497811317444, "median": 0.18024154007434845, "p05": 0.15453973412513733, "p95": 0.2030555009841919 }, { "label": "completed_group_2", "mean": 0.1307673156261444, "median": 0.13107559084892273, "p05": 0.11463287472724915, "p95": 0.14630235731601715 }, { "label": "completed_group_3", "mean": 0.1554446816444397, "median": 0.15516987442970276, "p05": 0.13627474009990692, "p95": 0.1754152774810791 }, { "label": "current_group_partial", "mean": 0.31774693727493286, "median": 0.319039523601532, "p05": 0.27606821060180664, "p95": 0.35650834441185 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.5068919658660889, "group": 4, "latest_source_mass": 0.3298327624797821, "layer": 15, "max_source_mass": 0.3298327624797821, "mixer_index": 29, "normalized_entropy": 0.9362846209998111, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.20646697282791138, "median": 0.18775638937950134, "p05": 0.0806753933429718, "p95": 0.4274584949016571 }, { "label": "completed_group_1", "mean": 0.1197461187839508, "median": 0.11989086866378784, "p05": 0.0809398666024208, "p95": 0.15743419528007507 }, { "label": "completed_group_2", "mean": 0.19215428829193115, "median": 0.19620640575885773, "p05": 0.1148109883069992, "p95": 0.2491212636232376 }, { "label": "completed_group_3", "mean": 0.15179988741874695, "median": 0.15212899446487427, "p05": 0.10697861760854721, "p95": 0.19542087614536285 }, { "label": "current_group_partial", "mean": 0.3298327624797821, "median": 0.32837340235710144, "p05": 0.22746729850769043, "p95": 0.4339301884174347 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.5435223579406738, "group": 4, "latest_source_mass": 0.3224271535873413, "layer": 16, "max_source_mass": 0.3224271535873413, "mixer_index": 30, "normalized_entropy": 0.9590443632623663, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.22410975396633148, "median": 0.22760415077209473, "p05": 0.14593973755836487, "p95": 0.30093055963516235 }, { "label": "completed_group_1", "mean": 0.16986176371574402, "median": 0.16963240504264832, "p05": 0.13569067418575287, "p95": 0.20651353895664215 }, { "label": "completed_group_2", "mean": 0.13080403208732605, "median": 0.13075244426727295, "p05": 0.10945598781108856, "p95": 0.15187400579452515 }, { "label": "completed_group_3", "mean": 0.15279728174209595, "median": 0.15036612749099731, "p05": 0.12810495495796204, "p95": 0.18468116223812103 }, { "label": "current_group_partial", "mean": 0.3224271535873413, "median": 0.3164973258972168, "p05": 0.2600978910923004, "p95": 0.40530598163604736 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.4949920177459717, "group": 4, "latest_source_mass": 0.31847020983695984, "layer": 16, "max_source_mass": 0.31847020983695984, "mixer_index": 31, "normalized_entropy": 0.9288907675133354, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.27174079418182373, "median": 0.2585451304912567, "p05": 0.13639721274375916, "p95": 0.4698348045349121 }, { "label": "completed_group_1", "mean": 0.12178683280944824, "median": 0.11714950203895569, "p05": 0.08594125509262085, "p95": 0.1723237782716751 }, { "label": "completed_group_2", "mean": 0.1529826819896698, "median": 0.15223106741905212, "p05": 0.10371473431587219, "p95": 0.203575000166893 }, { "label": "completed_group_3", "mean": 0.1350194811820984, "median": 0.13363957405090332, "p05": 0.08964549005031586, "p95": 0.18468207120895386 }, { "label": "current_group_partial", "mean": 0.31847020983695984, "median": 0.3185216188430786, "p05": 0.21444711089134216, "p95": 0.4301546812057495 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.544264793395996, "group": 5, "latest_source_mass": 0.2861037850379944, "layer": 17, "max_source_mass": 0.2861037850379944, "mixer_index": 32, "normalized_entropy": 0.9595056643474137, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.24016115069389343, "median": 0.23868846893310547, "p05": 0.18435582518577576, "p95": 0.2983453571796417 }, { "label": "completed_group_1", "mean": 0.20730282366275787, "median": 0.1998077780008316, "p05": 0.16255050897598267, "p95": 0.28107208013534546 }, { "label": "completed_group_2", "mean": 0.09692573547363281, "median": 0.0967404842376709, "p05": 0.08098706603050232, "p95": 0.11221462488174438 }, { "label": "completed_group_3", "mean": 0.1695065200328827, "median": 0.1696585714817047, "p05": 0.136396586894989, "p95": 0.20130634307861328 }, { "label": "completed_group_4", "mean": 0.2861037850379944, "median": 0.28883200883865356, "p05": 0.22401469945907593, "p95": 0.3409370183944702 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.686682939529419, "group": 5, "latest_source_mass": 0.12281867861747742, "layer": 17, "max_source_mass": 0.2885604202747345, "mixer_index": 33, "normalized_entropy": 0.9413556721740635, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.16755425930023193, "median": 0.1327776312828064, "p05": 0.0790192037820816, "p95": 0.3667412996292114 }, { "label": "completed_group_1", "mean": 0.10962115228176117, "median": 0.10525335371494293, "p05": 0.07840172946453094, "p95": 0.15405339002609253 }, { "label": "completed_group_2", "mean": 0.1783454418182373, "median": 0.17896884679794312, "p05": 0.10802385956048965, "p95": 0.24048911035060883 }, { "label": "completed_group_3", "mean": 0.13310004770755768, "median": 0.13033035397529602, "p05": 0.0904831737279892, "p95": 0.17934636771678925 }, { "label": "completed_group_4", "mean": 0.2885604202747345, "median": 0.2831825315952301, "p05": 0.16967125236988068, "p95": 0.4189499616622925 }, { "label": "current_group_partial", "mean": 0.12281867861747742, "median": 0.12062528729438782, "p05": 0.07461707293987274, "p95": 0.1782219111919403 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.7070834636688232, "group": 5, "latest_source_mass": 0.2988377809524536, "layer": 18, "max_source_mass": 0.2988377809524536, "mixer_index": 34, "normalized_entropy": 0.9527414214834803, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.181177020072937, "median": 0.18419909477233887, "p05": 0.13624237477779388, "p95": 0.22192169725894928 }, { "label": "completed_group_1", "mean": 0.14557164907455444, "median": 0.144476979970932, "p05": 0.1293315887451172, "p95": 0.16486899554729462 }, { "label": "completed_group_2", "mean": 0.08318156749010086, "median": 0.0820261612534523, "p05": 0.07088497281074524, "p95": 0.0990293100476265 }, { "label": "completed_group_3", "mean": 0.11424209177494049, "median": 0.11267061531543732, "p05": 0.0979941189289093, "p95": 0.1353992223739624 }, { "label": "completed_group_4", "mean": 0.17698991298675537, "median": 0.17597860097885132, "p05": 0.14837387204170227, "p95": 0.21050144731998444 }, { "label": "current_group_partial", "mean": 0.2988377809524536, "median": 0.2989804744720459, "p05": 0.2721370458602905, "p95": 0.32572072744369507 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.7419235706329346, "group": 5, "latest_source_mass": 0.26016175746917725, "layer": 18, "max_source_mass": 0.26016175746917725, "mixer_index": 35, "normalized_entropy": 0.972186055410333, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1532188355922699, "median": 0.1451343297958374, "p05": 0.11514341086149216, "p95": 0.22103679180145264 }, { "label": "completed_group_1", "mean": 0.11420352011919022, "median": 0.11261284351348877, "p05": 0.0928458571434021, "p95": 0.14123988151550293 }, { "label": "completed_group_2", "mean": 0.16597424447536469, "median": 0.1655328869819641, "p05": 0.13871800899505615, "p95": 0.19611573219299316 }, { "label": "completed_group_3", "mean": 0.1250365972518921, "median": 0.12369821965694427, "p05": 0.10375828295946121, "p95": 0.14912742376327515 }, { "label": "completed_group_4", "mean": 0.18140508234500885, "median": 0.1796584278345108, "p05": 0.14511024951934814, "p95": 0.22286373376846313 }, { "label": "current_group_partial", "mean": 0.26016175746917725, "median": 0.26266711950302124, "p05": 0.1889389455318451, "p95": 0.3234381675720215 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.7127439975738525, "group": 5, "latest_source_mass": 0.2907913327217102, "layer": 19, "max_source_mass": 0.2907913327217102, "mixer_index": 36, "normalized_entropy": 0.9559006256078307, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.18731768429279327, "median": 0.19042012095451355, "p05": 0.13879702985286713, "p95": 0.23792000114917755 }, { "label": "completed_group_1", "mean": 0.16715484857559204, "median": 0.1667211651802063, "p05": 0.14574480056762695, "p95": 0.19026139378547668 }, { "label": "completed_group_2", "mean": 0.08496320247650146, "median": 0.08452782034873962, "p05": 0.07410907745361328, "p95": 0.097386933863163 }, { "label": "completed_group_3", "mean": 0.11362075805664062, "median": 0.11248823255300522, "p05": 0.0977015346288681, "p95": 0.13219426572322845 }, { "label": "completed_group_4", "mean": 0.15615218877792358, "median": 0.15540224313735962, "p05": 0.1289147436618805, "p95": 0.18598109483718872 }, { "label": "current_group_partial", "mean": 0.2907913327217102, "median": 0.2900415062904358, "p05": 0.25822532176971436, "p95": 0.32475030422210693 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.6076834201812744, "group": 5, "latest_source_mass": 0.3738824725151062, "layer": 19, "max_source_mass": 0.3738824725151062, "mixer_index": 37, "normalized_entropy": 0.8972652009334232, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.15032559633255005, "median": 0.12544110417366028, "p05": 0.06976518034934998, "p95": 0.35840943455696106 }, { "label": "completed_group_1", "mean": 0.08624927699565887, "median": 0.08602266013622284, "p05": 0.061998721212148666, "p95": 0.11138717085123062 }, { "label": "completed_group_2", "mean": 0.15736976265907288, "median": 0.15802210569381714, "p05": 0.10121746361255646, "p95": 0.20687000453472137 }, { "label": "completed_group_3", "mean": 0.08584291487932205, "median": 0.0827617198228836, "p05": 0.060561805963516235, "p95": 0.11351694166660309 }, { "label": "completed_group_4", "mean": 0.14632999897003174, "median": 0.1426895558834076, "p05": 0.10106660425662994, "p95": 0.20179018378257751 }, { "label": "current_group_partial", "mean": 0.3738824725151062, "median": 0.3799295425415039, "p05": 0.23631010949611664, "p95": 0.4884527325630188 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.709290862083435, "group": 5, "latest_source_mass": 0.2901708781719208, "layer": 20, "max_source_mass": 0.2901708781719208, "mixer_index": 38, "normalized_entropy": 0.9539733939957076, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.18991519510746002, "median": 0.1924753338098526, "p05": 0.14201581478118896, "p95": 0.23709386587142944 }, { "label": "completed_group_1", "mean": 0.17091524600982666, "median": 0.17006167769432068, "p05": 0.15297070145606995, "p95": 0.19153237342834473 }, { "label": "completed_group_2", "mean": 0.07822999358177185, "median": 0.07778562605381012, "p05": 0.06707640737295151, "p95": 0.09089364111423492 }, { "label": "completed_group_3", "mean": 0.116765096783638, "median": 0.11516137421131134, "p05": 0.10162825882434845, "p95": 0.13653048872947693 }, { "label": "completed_group_4", "mean": 0.1540035903453827, "median": 0.15287795662879944, "p05": 0.12593276798725128, "p95": 0.18590238690376282 }, { "label": "current_group_partial", "mean": 0.2901708781719208, "median": 0.29104921221733093, "p05": 0.25643545389175415, "p95": 0.3205650746822357 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.6690887212753296, "group": 5, "latest_source_mass": 0.3342471718788147, "layer": 20, "max_source_mass": 0.3342471718788147, "mixer_index": 39, "normalized_entropy": 0.9315361520005944, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.19029226899147034, "median": 0.19644704461097717, "p05": 0.09456793963909149, "p95": 0.30468082427978516 }, { "label": "completed_group_1", "mean": 0.1100241169333458, "median": 0.10808776319026947, "p05": 0.09035106748342514, "p95": 0.1359245330095291 }, { "label": "completed_group_2", "mean": 0.11471512168645859, "median": 0.11494918912649155, "p05": 0.08799737691879272, "p95": 0.14024731516838074 }, { "label": "completed_group_3", "mean": 0.09999909996986389, "median": 0.09682698547840118, "p05": 0.07921088486909866, "p95": 0.1264391839504242 }, { "label": "completed_group_4", "mean": 0.1507222205400467, "median": 0.1476232260465622, "p05": 0.11397356539964676, "p95": 0.19991227984428406 }, { "label": "current_group_partial", "mean": 0.3342471718788147, "median": 0.33802270889282227, "p05": 0.2548266053199768, "p95": 0.40130624175071716 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.7458693981170654, "group": 6, "latest_source_mass": 0.2361394464969635, "layer": 21, "max_source_mass": 0.2361394464969635, "mixer_index": 40, "normalized_entropy": 0.9743882636597644, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.20528709888458252, "median": 0.2140858769416809, "p05": 0.13168928027153015, "p95": 0.27098023891448975 }, { "label": "completed_group_1", "mean": 0.1722964197397232, "median": 0.17238952219486237, "p05": 0.15053534507751465, "p95": 0.19365043938159943 }, { "label": "completed_group_2", "mean": 0.10435444861650467, "median": 0.10361101478338242, "p05": 0.08888213336467743, "p95": 0.12212536484003067 }, { "label": "completed_group_3", "mean": 0.12075909972190857, "median": 0.1180175170302391, "p05": 0.10358668118715286, "p95": 0.14458929002285004 }, { "label": "completed_group_4", "mean": 0.16116347908973694, "median": 0.15841388702392578, "p05": 0.1337081342935562, "p95": 0.19791515171527863 }, { "label": "completed_group_5", "mean": 0.2361394464969635, "median": 0.23709020018577576, "p05": 0.20446214079856873, "p95": 0.26509594917297363 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.4612125158309937, "group": 6, "latest_source_mass": 0.5543455481529236, "layer": 21, "max_source_mass": 0.5543455481529236, "mixer_index": 41, "normalized_entropy": 0.7509146897354808, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.08934466540813446, "median": 0.07911523431539536, "p05": 0.04488275945186615, "p95": 0.1737232506275177 }, { "label": "completed_group_1", "mean": 0.054379627108573914, "median": 0.05344422906637192, "p05": 0.04221750423312187, "p95": 0.06924191862344742 }, { "label": "completed_group_2", "mean": 0.07567165046930313, "median": 0.07438549399375916, "p05": 0.05811355262994766, "p95": 0.09632256627082825 }, { "label": "completed_group_3", "mean": 0.05758609622716904, "median": 0.05612001568078995, "p05": 0.044792283326387405, "p95": 0.07347827404737473 }, { "label": "completed_group_4", "mean": 0.07884950935840607, "median": 0.07757881283760071, "p05": 0.05758974328637123, "p95": 0.10452556610107422 }, { "label": "completed_group_5", "mean": 0.08982288837432861, "median": 0.08912864327430725, "p05": 0.0663854256272316, "p95": 0.11604885756969452 }, { "label": "current_group_partial", "mean": 0.5543455481529236, "median": 0.5596089363098145, "p05": 0.45455610752105713, "p95": 0.6367484331130981 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.7241814136505127, "group": 6, "latest_source_mass": 0.39484071731567383, "layer": 22, "max_source_mass": 0.39484071731567383, "mixer_index": 42, "normalized_entropy": 0.886053970419732, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.13360010087490082, "median": 0.14072942733764648, "p05": 0.07699814438819885, "p95": 0.17506597936153412 }, { "label": "completed_group_1", "mean": 0.10609637200832367, "median": 0.10586297512054443, "p05": 0.08648421615362167, "p95": 0.12602579593658447 }, { "label": "completed_group_2", "mean": 0.06363911926746368, "median": 0.06324908882379532, "p05": 0.054980404675006866, "p95": 0.07316673547029495 }, { "label": "completed_group_3", "mean": 0.07426095008850098, "median": 0.07362888753414154, "p05": 0.06318790465593338, "p95": 0.08665015548467636 }, { "label": "completed_group_4", "mean": 0.09383076429367065, "median": 0.09294568747282028, "p05": 0.07638302445411682, "p95": 0.11445100605487823 }, { "label": "completed_group_5", "mean": 0.13373197615146637, "median": 0.13327741622924805, "p05": 0.11399447172880173, "p95": 0.15641780197620392 }, { "label": "current_group_partial", "mean": 0.39484071731567383, "median": 0.3879151940345764, "p05": 0.3302172124385834, "p95": 0.4852263331413269 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.3979300260543823, "group": 6, "latest_source_mass": 0.5830056071281433, "layer": 22, "max_source_mass": 0.5830056071281433, "mixer_index": 43, "normalized_entropy": 0.7183939231382495, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.08959861099720001, "median": 0.07544538378715515, "p05": 0.04165133833885193, "p95": 0.20099139213562012 }, { "label": "completed_group_1", "mean": 0.05414731800556183, "median": 0.05336771905422211, "p05": 0.04069708660244942, "p95": 0.07046990841627121 }, { "label": "completed_group_2", "mean": 0.07668881118297577, "median": 0.07607124000787735, "p05": 0.0555989071726799, "p95": 0.09980960935354233 }, { "label": "completed_group_3", "mean": 0.05603286623954773, "median": 0.054573021829128265, "p05": 0.04162490367889404, "p95": 0.0730118378996849 }, { "label": "completed_group_4", "mean": 0.06979228556156158, "median": 0.06884932518005371, "p05": 0.04576495662331581, "p95": 0.09806846082210541 }, { "label": "completed_group_5", "mean": 0.07073447108268738, "median": 0.07023533433675766, "p05": 0.04956750199198723, "p95": 0.09379896521568298 }, { "label": "current_group_partial", "mean": 0.5830056071281433, "median": 0.5862616896629333, "p05": 0.4658474922180176, "p95": 0.6849814057350159 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.7459560632705688, "group": 6, "latest_source_mass": 0.37691548466682434, "layer": 23, "max_source_mass": 0.37691548466682434, "mixer_index": 44, "normalized_entropy": 0.897243926765161, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.14318783581256866, "median": 0.1493408977985382, "p05": 0.08292607218027115, "p95": 0.19464610517024994 }, { "label": "completed_group_1", "mean": 0.1094653457403183, "median": 0.10858799517154694, "p05": 0.09432064741849899, "p95": 0.1279931515455246 }, { "label": "completed_group_2", "mean": 0.06349228322505951, "median": 0.06325754523277283, "p05": 0.05439673364162445, "p95": 0.0737837627530098 }, { "label": "completed_group_3", "mean": 0.07859592139720917, "median": 0.0775163471698761, "p05": 0.06603655964136124, "p95": 0.09357087314128876 }, { "label": "completed_group_4", "mean": 0.09567482024431229, "median": 0.09468592703342438, "p05": 0.07671141624450684, "p95": 0.11774884164333344 }, { "label": "completed_group_5", "mean": 0.13266831636428833, "median": 0.13199329376220703, "p05": 0.11244359612464905, "p95": 0.15504513680934906 }, { "label": "current_group_partial", "mean": 0.37691548466682434, "median": 0.37180638313293457, "p05": 0.31207239627838135, "p95": 0.4595233201980591 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.4055393934249878, "group": 6, "latest_source_mass": 0.5772827863693237, "layer": 23, "max_source_mass": 0.5772827863693237, "mixer_index": 45, "normalized_entropy": 0.7223043644164862, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.10154955089092255, "median": 0.08766824007034302, "p05": 0.04901552200317383, "p95": 0.20287738740444183 }, { "label": "completed_group_1", "mean": 0.05619276314973831, "median": 0.055477917194366455, "p05": 0.041608311235904694, "p95": 0.07371112704277039 }, { "label": "completed_group_2", "mean": 0.07791627198457718, "median": 0.07736755907535553, "p05": 0.055920060724020004, "p95": 0.10158214718103409 }, { "label": "completed_group_3", "mean": 0.053915880620479584, "median": 0.05215863138437271, "p05": 0.03798118978738785, "p95": 0.07156119495630264 }, { "label": "completed_group_4", "mean": 0.06398355215787888, "median": 0.06229078769683838, "p05": 0.040616974234580994, "p95": 0.09270931780338287 }, { "label": "completed_group_5", "mean": 0.06915920227766037, "median": 0.0684593990445137, "p05": 0.046986356377601624, "p95": 0.09408729523420334 }, { "label": "current_group_partial", "mean": 0.5772827863693237, "median": 0.5824689269065857, "p05": 0.4518115520477295, "p95": 0.6926202178001404 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.7853620052337646, "group": 6, "latest_source_mass": 0.3407441973686218, "layer": 24, "max_source_mass": 0.3407441973686218, "mixer_index": 46, "normalized_entropy": 0.9174945750195659, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.15284113585948944, "median": 0.1593906283378601, "p05": 0.08678612112998962, "p95": 0.211342453956604 }, { "label": "completed_group_1", "mean": 0.12206703424453735, "median": 0.12153112143278122, "p05": 0.1060115247964859, "p95": 0.1399766057729721 }, { "label": "completed_group_2", "mean": 0.06508444249629974, "median": 0.06482555717229843, "p05": 0.055350616574287415, "p95": 0.07557765394449234 }, { "label": "completed_group_3", "mean": 0.08347313106060028, "median": 0.08190013468265533, "p05": 0.06966532766819, "p95": 0.10050469636917114 }, { "label": "completed_group_4", "mean": 0.09811970591545105, "median": 0.09664031863212585, "p05": 0.07688035070896149, "p95": 0.12369741499423981 }, { "label": "completed_group_5", "mean": 0.13767032325267792, "median": 0.1373325139284134, "p05": 0.11642646044492722, "p95": 0.1606694608926773 }, { "label": "current_group_partial", "mean": 0.3407441973686218, "median": 0.33805108070373535, "p05": 0.271301805973053, "p95": 0.41781002283096313 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.509842872619629, "group": 6, "latest_source_mass": 0.5227814316749573, "layer": 24, "max_source_mass": 0.5227814316749573, "mixer_index": 47, "normalized_entropy": 0.7759057494780099, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.11788781732320786, "median": 0.10097194463014603, "p05": 0.06265684217214584, "p95": 0.2408325970172882 }, { "label": "completed_group_1", "mean": 0.06822571158409119, "median": 0.0673636943101883, "p05": 0.05209728330373764, "p95": 0.08648915588855743 }, { "label": "completed_group_2", "mean": 0.08556114137172699, "median": 0.08463940769433975, "p05": 0.06234566867351532, "p95": 0.1125786080956459 }, { "label": "completed_group_3", "mean": 0.06067090481519699, "median": 0.058974385261535645, "p05": 0.043267179280519485, "p95": 0.07997901737689972 }, { "label": "completed_group_4", "mean": 0.07228855788707733, "median": 0.06972275674343109, "p05": 0.04689513146877289, "p95": 0.106512151658535 }, { "label": "completed_group_5", "mean": 0.07258446514606476, "median": 0.07082539796829224, "p05": 0.050949547439813614, "p95": 0.09970614314079285 }, { "label": "current_group_partial", "mean": 0.5227814316749573, "median": 0.5290273427963257, "p05": 0.38773614168167114, "p95": 0.6392589807510376 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.839367389678955, "group": 7, "latest_source_mass": 0.29583215713500977, "layer": 25, "max_source_mass": 0.29583215713500977, "mixer_index": 48, "normalized_entropy": 0.9452478525649903, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.15906387567520142, "median": 0.15846745669841766, "p05": 0.09519976377487183, "p95": 0.24517810344696045 }, { "label": "completed_group_1", "mean": 0.13139380514621735, "median": 0.1308426856994629, "p05": 0.11478274315595627, "p95": 0.15000829100608826 }, { "label": "completed_group_2", "mean": 0.07717745751142502, "median": 0.07715500891208649, "p05": 0.06616044044494629, "p95": 0.08791667222976685 }, { "label": "completed_group_3", "mean": 0.09607303142547607, "median": 0.09436552226543427, "p05": 0.08152313530445099, "p95": 0.11384639143943787 }, { "label": "completed_group_4", "mean": 0.1107850894331932, "median": 0.1089654266834259, "p05": 0.08814431726932526, "p95": 0.139452263712883 }, { "label": "completed_group_5", "mean": 0.12967455387115479, "median": 0.1299237608909607, "p05": 0.10832332074642181, "p95": 0.15138982236385345 }, { "label": "completed_group_6", "mean": 0.29583215713500977, "median": 0.29647016525268555, "p05": 0.2276897132396698, "p95": 0.3604946434497833 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.5841834545135498, "group": 7, "latest_source_mass": 0.40767741203308105, "layer": 25, "max_source_mass": 0.40767741203308105, "mixer_index": 49, "normalized_entropy": 0.7618312045616817, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.07896879315376282, "median": 0.06809917092323303, "p05": 0.025932887569069862, "p95": 0.19368338584899902 }, { "label": "completed_group_1", "mean": 0.048752427101135254, "median": 0.04767637699842453, "p05": 0.034443892538547516, "p95": 0.0667153000831604 }, { "label": "completed_group_2", "mean": 0.055707164108753204, "median": 0.0549345538020134, "p05": 0.038696639239788055, "p95": 0.0750277042388916 }, { "label": "completed_group_3", "mean": 0.04139826446771622, "median": 0.03878612071275711, "p05": 0.026673054322600365, "p95": 0.05746251344680786 }, { "label": "completed_group_4", "mean": 0.04449018836021423, "median": 0.04196460545063019, "p05": 0.026162121444940567, "p95": 0.07164917886257172 }, { "label": "completed_group_5", "mean": 0.04285799711942673, "median": 0.04109475016593933, "p05": 0.029393533244729042, "p95": 0.06277710199356079 }, { "label": "completed_group_6", "mean": 0.2801477909088135, "median": 0.2763082981109619, "p05": 0.1455708146095276, "p95": 0.4254714846611023 }, { "label": "current_group_partial", "mean": 0.40767741203308105, "median": 0.4120897948741913, "p05": 0.21399420499801636, "p95": 0.5853031277656555 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.8125314712524414, "group": 7, "latest_source_mass": 0.3700707256793976, "layer": 26, "max_source_mass": 0.3700707256793976, "mixer_index": 50, "normalized_entropy": 0.8716433883436914, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.10889513790607452, "median": 0.11333495378494263, "p05": 0.04850132763385773, "p95": 0.1620434820652008 }, { "label": "completed_group_1", "mean": 0.08656036853790283, "median": 0.08640051633119583, "p05": 0.0650845319032669, "p95": 0.1084962785243988 }, { "label": "completed_group_2", "mean": 0.048123255372047424, "median": 0.04800213500857353, "p05": 0.03576180338859558, "p95": 0.06107594817876816 }, { "label": "completed_group_3", "mean": 0.06561170518398285, "median": 0.06379580497741699, "p05": 0.050968848168849945, "p95": 0.08332888036966324 }, { "label": "completed_group_4", "mean": 0.06917259097099304, "median": 0.06724178791046143, "p05": 0.050207968801259995, "p95": 0.09373342245817184 }, { "label": "completed_group_5", "mean": 0.08286689221858978, "median": 0.08183546364307404, "p05": 0.06535022705793381, "p95": 0.10340933501720428 }, { "label": "completed_group_6", "mean": 0.16869932413101196, "median": 0.16509854793548584, "p05": 0.11059704422950745, "p95": 0.23654375970363617 }, { "label": "current_group_partial", "mean": 0.3700707256793976, "median": 0.3694133758544922, "p05": 0.23441238701343536, "p95": 0.5025027990341187 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.50089693069458, "group": 7, "latest_source_mass": 0.49096518754959106, "layer": 26, "max_source_mass": 0.49096518754959106, "mixer_index": 51, "normalized_entropy": 0.7217788529328457, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.07974998652935028, "median": 0.07402504235506058, "p05": 0.04579866677522659, "p95": 0.1359037309885025 }, { "label": "completed_group_1", "mean": 0.04166398197412491, "median": 0.04038450866937637, "p05": 0.027992289513349533, "p95": 0.05969971418380737 }, { "label": "completed_group_2", "mean": 0.04607272148132324, "median": 0.044438350945711136, "p05": 0.030038591474294662, "p95": 0.06808309257030487 }, { "label": "completed_group_3", "mean": 0.03586764633655548, "median": 0.03317485377192497, "p05": 0.02203454077243805, "p95": 0.05572832003235817 }, { "label": "completed_group_4", "mean": 0.04223340004682541, "median": 0.03930673003196716, "p05": 0.02465122565627098, "p95": 0.06968094408512115 }, { "label": "completed_group_5", "mean": 0.04095568507909775, "median": 0.03919431194663048, "p05": 0.026239857077598572, "p95": 0.06058088690042496 }, { "label": "completed_group_6", "mean": 0.22249138355255127, "median": 0.2125401496887207, "p05": 0.10647299885749817, "p95": 0.3711881935596466 }, { "label": "current_group_partial", "mean": 0.49096518754959106, "median": 0.4988659620285034, "p05": 0.2878011167049408, "p95": 0.6676807403564453 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.8059929609298706, "group": 7, "latest_source_mass": 0.37139976024627686, "layer": 27, "max_source_mass": 0.37139976024627686, "mixer_index": 52, "normalized_entropy": 0.8684990295379666, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.10711923241615295, "median": 0.10972198843955994, "p05": 0.04896187782287598, "p95": 0.16201043128967285 }, { "label": "completed_group_1", "mean": 0.07651707530021667, "median": 0.07666032761335373, "p05": 0.05141128972172737, "p95": 0.10152198374271393 }, { "label": "completed_group_2", "mean": 0.058284737169742584, "median": 0.05812416970729828, "p05": 0.039273619651794434, "p95": 0.07745087146759033 }, { "label": "completed_group_3", "mean": 0.06609587371349335, "median": 0.06383530795574188, "p05": 0.04481930285692215, "p95": 0.09336265176534653 }, { "label": "completed_group_4", "mean": 0.07139429450035095, "median": 0.0694514587521553, "p05": 0.0485839769244194, "p95": 0.10034126043319702 }, { "label": "completed_group_5", "mean": 0.07493151724338531, "median": 0.07319007813930511, "p05": 0.051876723766326904, "p95": 0.10402432829141617 }, { "label": "completed_group_6", "mean": 0.17425751686096191, "median": 0.16937679052352905, "p05": 0.10449051856994629, "p95": 0.2624811828136444 }, { "label": "current_group_partial", "mean": 0.37139976024627686, "median": 0.3732905089855194, "p05": 0.23830299079418182, "p95": 0.5014542937278748 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.5622732639312744, "group": 7, "latest_source_mass": 0.4768889844417572, "layer": 27, "max_source_mass": 0.4768889844417572, "mixer_index": 53, "normalized_entropy": 0.7512946301290215, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.08775827288627625, "median": 0.08504067361354828, "p05": 0.0428270623087883, "p95": 0.144285649061203 }, { "label": "completed_group_1", "mean": 0.047765426337718964, "median": 0.04646281152963638, "p05": 0.031814731657505035, "p95": 0.06807165592908859 }, { "label": "completed_group_2", "mean": 0.053928617388010025, "median": 0.052262187004089355, "p05": 0.035437777638435364, "p95": 0.0784272700548172 }, { "label": "completed_group_3", "mean": 0.04293380305171013, "median": 0.03996016085147858, "p05": 0.02556057646870613, "p95": 0.06623870134353638 }, { "label": "completed_group_4", "mean": 0.04174989461898804, "median": 0.03903091698884964, "p05": 0.025013400241732597, "p95": 0.06869364529848099 }, { "label": "completed_group_5", "mean": 0.04734005779027939, "median": 0.04609068110585213, "p05": 0.029884062707424164, "p95": 0.06911227852106094 }, { "label": "completed_group_6", "mean": 0.20163494348526, "median": 0.19222964346408844, "p05": 0.09963390231132507, "p95": 0.3307305574417114 }, { "label": "current_group_partial", "mean": 0.4768889844417572, "median": 0.48383551836013794, "p05": 0.276212215423584, "p95": 0.6487575173377991 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.8757455348968506, "group": 7, "latest_source_mass": 0.3109493851661682, "layer": 28, "max_source_mass": 0.3109493851661682, "mixer_index": 54, "normalized_entropy": 0.9020429270551009, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.11563669890165329, "median": 0.12087491154670715, "p05": 0.05733206868171692, "p95": 0.1688953936100006 }, { "label": "completed_group_1", "mean": 0.08108697831630707, "median": 0.08185897022485733, "p05": 0.05175650864839554, "p95": 0.10753051191568375 }, { "label": "completed_group_2", "mean": 0.07280858606100082, "median": 0.07357072830200195, "p05": 0.04870978742837906, "p95": 0.0948689877986908 }, { "label": "completed_group_3", "mean": 0.07553227245807648, "median": 0.07362619787454605, "p05": 0.056317005306482315, "p95": 0.09756095707416534 }, { "label": "completed_group_4", "mean": 0.07606014609336853, "median": 0.07456053793430328, "p05": 0.05528895929455757, "p95": 0.1022825762629509 }, { "label": "completed_group_5", "mean": 0.06935927271842957, "median": 0.06843917071819305, "p05": 0.04978285729885101, "p95": 0.09167985618114471 }, { "label": "completed_group_6", "mean": 0.19856664538383484, "median": 0.19508783519268036, "p05": 0.1281973123550415, "p95": 0.27796900272369385 }, { "label": "current_group_partial", "mean": 0.3109493851661682, "median": 0.3105033040046692, "p05": 0.18172834813594818, "p95": 0.43916669487953186 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.6924852132797241, "group": 7, "latest_source_mass": 0.3315581679344177, "layer": 28, "max_source_mass": 0.3315581679344177, "mixer_index": 55, "normalized_entropy": 0.8139133413255193, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.08638577908277512, "median": 0.058608733117580414, "p05": 0.026544082909822464, "p95": 0.2740764319896698 }, { "label": "completed_group_1", "mean": 0.059912845492362976, "median": 0.05901291221380234, "p05": 0.03629481792449951, "p95": 0.08707985281944275 }, { "label": "completed_group_2", "mean": 0.06609142571687698, "median": 0.0647939145565033, "p05": 0.038691695779561996, "p95": 0.09703366458415985 }, { "label": "completed_group_3", "mean": 0.056745611131191254, "median": 0.05418796092271805, "p05": 0.032174739986658096, "p95": 0.08370660990476608 }, { "label": "completed_group_4", "mean": 0.06277773529291153, "median": 0.0600711889564991, "p05": 0.03501531109213829, "p95": 0.10094858705997467 }, { "label": "completed_group_5", "mean": 0.05352145433425903, "median": 0.05179818347096443, "p05": 0.032271772623062134, "p95": 0.08084158599376678 }, { "label": "completed_group_6", "mean": 0.2830069661140442, "median": 0.2759752869606018, "p05": 0.12981083989143372, "p95": 0.46204638481140137 }, { "label": "current_group_partial", "mean": 0.3315581679344177, "median": 0.3204721510410309, "p05": 0.13446375727653503, "p95": 0.5667697787284851 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.8980522155761719, "group": 8, "latest_source_mass": 0.20123262703418732, "layer": 29, "max_source_mass": 0.26602888107299805, "mixer_index": 56, "normalized_entropy": 0.9127701729200177, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.11603894829750061, "median": 0.09991352260112762, "p05": 0.030035395175218582, "p95": 0.2941743731498718 }, { "label": "completed_group_1", "mean": 0.08804173022508621, "median": 0.08880250155925751, "p05": 0.05664174258708954, "p95": 0.11779069900512695 }, { "label": "completed_group_2", "mean": 0.07975024729967117, "median": 0.07984748482704163, "p05": 0.05428915470838547, "p95": 0.10499341040849686 }, { "label": "completed_group_3", "mean": 0.09027000516653061, "median": 0.08898856490850449, "p05": 0.06189651042222977, "p95": 0.11996553838253021 }, { "label": "completed_group_4", "mean": 0.08430293202400208, "median": 0.0819794088602066, "p05": 0.05464346706867218, "p95": 0.12159879505634308 }, { "label": "completed_group_5", "mean": 0.07433462142944336, "median": 0.07287181913852692, "p05": 0.052520088851451874, "p95": 0.1010315865278244 }, { "label": "completed_group_6", "mean": 0.26602888107299805, "median": 0.2639757990837097, "p05": 0.15474572777748108, "p95": 0.3846370279788971 }, { "label": "completed_group_7", "mean": 0.20123262703418732, "median": 0.18140269815921783, "p05": 0.08419936150312424, "p95": 0.3782624900341034 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.7512003183364868, "group": 8, "latest_source_mass": 0.33482813835144043, "layer": 29, "max_source_mass": 0.33482813835144043, "mixer_index": 57, "normalized_entropy": 0.7970056117156376, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.07744461297988892, "median": 0.06970229744911194, "p05": 0.014401955530047417, "p95": 0.17144526541233063 }, { "label": "completed_group_1", "mean": 0.046086348593235016, "median": 0.04394831880927086, "p05": 0.028066720813512802, "p95": 0.0718703418970108 }, { "label": "completed_group_2", "mean": 0.05025433003902435, "median": 0.04861236736178398, "p05": 0.029644077643752098, "p95": 0.07761462777853012 }, { "label": "completed_group_3", "mean": 0.03748488053679466, "median": 0.03434944897890091, "p05": 0.020774798467755318, "p95": 0.05938909202814102 }, { "label": "completed_group_4", "mean": 0.04005061462521553, "median": 0.037498839199543, "p05": 0.019541850313544273, "p95": 0.06922487914562225 }, { "label": "completed_group_5", "mean": 0.041623204946517944, "median": 0.039552345871925354, "p05": 0.022081367671489716, "p95": 0.06821337342262268 }, { "label": "completed_group_6", "mean": 0.198423832654953, "median": 0.18318486213684082, "p05": 0.07548617571592331, "p95": 0.3721975088119507 }, { "label": "completed_group_7", "mean": 0.17380405962467194, "median": 0.1514919549226761, "p05": 0.05777408182621002, "p95": 0.36326974630355835 }, { "label": "current_group_partial", "mean": 0.33482813835144043, "median": 0.33602219820022583, "p05": 0.0795319601893425, "p95": 0.5979493856430054 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.0160694122314453, "group": 8, "latest_source_mass": 0.21711516380310059, "layer": 30, "max_source_mass": 0.21711516380310059, "mixer_index": 58, "normalized_entropy": 0.9175527313077866, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.09385271370410919, "median": 0.09211117774248123, "p05": 0.032377734780311584, "p95": 0.16496515274047852 }, { "label": "completed_group_1", "mean": 0.06208505108952522, "median": 0.06140740588307381, "p05": 0.0407925546169281, "p95": 0.0863042101264 }, { "label": "completed_group_2", "mean": 0.06560768187046051, "median": 0.06516212224960327, "p05": 0.041897669434547424, "p95": 0.09085705131292343 }, { "label": "completed_group_3", "mean": 0.07029052078723907, "median": 0.06780634075403214, "p05": 0.04846132919192314, "p95": 0.09917552024126053 }, { "label": "completed_group_4", "mean": 0.06935097277164459, "median": 0.06708167493343353, "p05": 0.04668639227747917, "p95": 0.09815650433301926 }, { "label": "completed_group_5", "mean": 0.056832700967788696, "median": 0.05595892667770386, "p05": 0.038822539150714874, "p95": 0.0771557092666626 }, { "label": "completed_group_6", "mean": 0.19425815343856812, "median": 0.19145764410495758, "p05": 0.12249615788459778, "p95": 0.27496030926704407 }, { "label": "completed_group_7", "mean": 0.17060703039169312, "median": 0.16317546367645264, "p05": 0.09239593893289566, "p95": 0.2726203501224518 }, { "label": "current_group_partial", "mean": 0.21711516380310059, "median": 0.21198657155036926, "p05": 0.11019522696733475, "p95": 0.3391069769859314 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 1.7837426662445068, "group": 8, "latest_source_mass": 0.29026859998703003, "layer": 30, "max_source_mass": 0.29026859998703003, "mixer_index": 59, "normalized_entropy": 0.8118162725118464, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.08836327493190765, "median": 0.07904411852359772, "p05": 0.02631576545536518, "p95": 0.17534467577934265 }, { "label": "completed_group_1", "mean": 0.04953558370471001, "median": 0.04658038914203644, "p05": 0.025181813165545464, "p95": 0.08318150043487549 }, { "label": "completed_group_2", "mean": 0.04813014343380928, "median": 0.045740917325019836, "p05": 0.026714935898780823, "p95": 0.07843323051929474 }, { "label": "completed_group_3", "mean": 0.0373263955116272, "median": 0.03401809185743332, "p05": 0.01990434154868126, "p95": 0.06290259957313538 }, { "label": "completed_group_4", "mean": 0.037015400826931, "median": 0.03405017405748367, "p05": 0.018646355718374252, "p95": 0.06544196605682373 }, { "label": "completed_group_5", "mean": 0.03390160948038101, "median": 0.03198609501123428, "p05": 0.018189728260040283, "p95": 0.05570102855563164 }, { "label": "completed_group_6", "mean": 0.1840541511774063, "median": 0.17327937483787537, "p05": 0.07966600358486176, "p95": 0.32982927560806274 }, { "label": "completed_group_7", "mean": 0.23140482604503632, "median": 0.22406929731369019, "p05": 0.08831030130386353, "p95": 0.40350067615509033 }, { "label": "current_group_partial", "mean": 0.29026859998703003, "median": 0.2745037376880646, "p05": 0.10280418395996094, "p95": 0.5399085879325867 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 1.9973840713500977, "group": 8, "latest_source_mass": 0.19101998209953308, "layer": 31, "max_source_mass": 0.20831164717674255, "mixer_index": 60, "normalized_entropy": 0.9090486661912383, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.10261696577072144, "median": 0.10389091819524765, "p05": 0.01371697522699833, "p95": 0.21163305640220642 }, { "label": "completed_group_1", "mean": 0.06039813905954361, "median": 0.06085003539919853, "p05": 0.026107093319296837, "p95": 0.09348798543214798 }, { "label": "completed_group_2", "mean": 0.06671924144029617, "median": 0.06656201928853989, "p05": 0.034684643149375916, "p95": 0.09819839894771576 }, { "label": "completed_group_3", "mean": 0.08377155661582947, "median": 0.0804532840847969, "p05": 0.05448959022760391, "p95": 0.12039553374052048 }, { "label": "completed_group_4", "mean": 0.07002593576908112, "median": 0.06737451255321503, "p05": 0.044841229915618896, "p95": 0.10655929893255234 }, { "label": "completed_group_5", "mean": 0.05768897384405136, "median": 0.056077782064676285, "p05": 0.039495617151260376, "p95": 0.08242568373680115 }, { "label": "completed_group_6", "mean": 0.20831164717674255, "median": 0.2021285891532898, "p05": 0.11652229726314545, "p95": 0.31706833839416504 }, { "label": "completed_group_7", "mean": 0.1594475358724594, "median": 0.1546279788017273, "p05": 0.06593266129493713, "p95": 0.2735573351383209 }, { "label": "current_group_partial", "mean": 0.19101998209953308, "median": 0.1713225245475769, "p05": 0.06408411264419556, "p95": 0.3850628137588501 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 1.969987154006958, "group": 8, "latest_source_mass": 0.16864418983459473, "layer": 31, "max_source_mass": 0.24503237009048462, "mixer_index": 61, "normalized_entropy": 0.8965797917640489, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.08120635151863098, "median": 0.061164144426584244, "p05": 0.0259944386780262, "p95": 0.23622018098831177 }, { "label": "completed_group_1", "mean": 0.07894632965326309, "median": 0.07658770680427551, "p05": 0.0496862456202507, "p95": 0.11596276611089706 }, { "label": "completed_group_2", "mean": 0.08615730702877045, "median": 0.08467075228691101, "p05": 0.050409119576215744, "p95": 0.12811316549777985 }, { "label": "completed_group_3", "mean": 0.07609779387712479, "median": 0.0722871869802475, "p05": 0.04099677503108978, "p95": 0.11950768530368805 }, { "label": "completed_group_4", "mean": 0.058728985488414764, "median": 0.056080836802721024, "p05": 0.03323862701654434, "p95": 0.09344407916069031 }, { "label": "completed_group_5", "mean": 0.05076971277594566, "median": 0.04946017265319824, "p05": 0.03047419898211956, "p95": 0.07563400268554688 }, { "label": "completed_group_6", "mean": 0.24503237009048462, "median": 0.24110376834869385, "p05": 0.10613486170768738, "p95": 0.3991503119468689 }, { "label": "completed_group_7", "mean": 0.1544169783592224, "median": 0.1400495171546936, "p05": 0.06199025735259056, "p95": 0.2995939254760742 }, { "label": "current_group_partial", "mean": 0.16864418983459473, "median": 0.15222924947738647, "p05": 0.0543956533074379, "p95": 0.3329208791255951 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.071648359298706, "group": 8, "latest_source_mass": 0.11520501971244812, "layer": 32, "max_source_mass": 0.20955611765384674, "mixer_index": 62, "normalized_entropy": 0.9428478002054053, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.10569609701633453, "median": 0.08689188957214355, "p05": 0.03173588588833809, "p95": 0.24571751058101654 }, { "label": "completed_group_1", "mean": 0.09922641515731812, "median": 0.09795533865690231, "p05": 0.06407676637172699, "p95": 0.14120742678642273 }, { "label": "completed_group_2", "mean": 0.08180510997772217, "median": 0.080217644572258, "p05": 0.054607540369033813, "p95": 0.11442367732524872 }, { "label": "completed_group_3", "mean": 0.10646799206733704, "median": 0.10510502755641937, "p05": 0.07415887713432312, "p95": 0.14216329157352448 }, { "label": "completed_group_4", "mean": 0.0826798751950264, "median": 0.08022409677505493, "p05": 0.05491747707128525, "p95": 0.11860857903957367 }, { "label": "completed_group_5", "mean": 0.06615597754716873, "median": 0.06485182791948318, "p05": 0.04477996751666069, "p95": 0.09238749742507935 }, { "label": "completed_group_6", "mean": 0.20955611765384674, "median": 0.206287682056427, "p05": 0.129981130361557, "p95": 0.2996031939983368 }, { "label": "completed_group_7", "mean": 0.13320739567279816, "median": 0.12465103715658188, "p05": 0.06303662061691284, "p95": 0.23065781593322754 }, { "label": "current_group_partial", "mean": 0.11520501971244812, "median": 0.10386130958795547, "p05": 0.03912060335278511, "p95": 0.22735844552516937 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 1.9781701564788818, "group": 8, "latest_source_mass": 0.16112206876277924, "layer": 32, "max_source_mass": 0.22251944243907928, "mixer_index": 63, "normalized_entropy": 0.9003040366848136, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.09446848928928375, "median": 0.074150450527668, "p05": 0.042592622339725494, "p95": 0.19443637132644653 }, { "label": "completed_group_1", "mean": 0.07056106626987457, "median": 0.06868506222963333, "p05": 0.04818308353424072, "p95": 0.0982733890414238 }, { "label": "completed_group_2", "mean": 0.08033448457717896, "median": 0.08047192543745041, "p05": 0.048296794295310974, "p95": 0.11253741383552551 }, { "label": "completed_group_3", "mean": 0.0658978745341301, "median": 0.06256154924631119, "p05": 0.04223886504769325, "p95": 0.0967884361743927 }, { "label": "completed_group_4", "mean": 0.058234501630067825, "median": 0.0563921295106411, "p05": 0.03529864549636841, "p95": 0.08846980333328247 }, { "label": "completed_group_5", "mean": 0.04316519945859909, "median": 0.042268507182598114, "p05": 0.029443470761179924, "p95": 0.05996086448431015 }, { "label": "completed_group_6", "mean": 0.2036968618631363, "median": 0.19645483791828156, "p05": 0.09398645162582397, "p95": 0.3395216464996338 }, { "label": "completed_group_7", "mean": 0.22251944243907928, "median": 0.2130982130765915, "p05": 0.11867167800664902, "p95": 0.35465875267982483 }, { "label": "current_group_partial", "mean": 0.16112206876277924, "median": 0.14023703336715698, "p05": 0.06008770316839218, "p95": 0.3247341811656952 } ], "sources": 9 }, { "branch": "output", "entropy_mean": 1.6036605834960938, "group": null, "latest_source_mass": 0.31639498472213745, "layer": null, "max_source_mass": 0.3412485122680664, "mixer_index": null, "normalized_entropy": 0.7298573846467136, "offset": null, "source_summaries": [ { "label": "embedding", "mean": 0.09759747982025146, "median": 0.09915056824684143, "p05": 0.025461740791797638, "p95": 0.16398483514785767 }, { "label": "completed_group_1", "mean": 0.04215940088033676, "median": 0.04050712287425995, "p05": 0.025720715522766113, "p95": 0.06398092210292816 }, { "label": "completed_group_2", "mean": 0.041145481169223785, "median": 0.03881259262561798, "p05": 0.023510465398430824, "p95": 0.0662468671798706 }, { "label": "completed_group_3", "mean": 0.029554028064012527, "median": 0.027420252561569214, "p05": 0.01627083122730255, "p95": 0.04924973472952843 }, { "label": "completed_group_4", "mean": 0.0289996936917305, "median": 0.027226522564888, "p05": 0.01560397818684578, "p95": 0.048351090401411057 }, { "label": "completed_group_5", "mean": 0.03647579252719879, "median": 0.0347352996468544, "p05": 0.020744025707244873, "p95": 0.058257561177015305 }, { "label": "completed_group_6", "mean": 0.06642461568117142, "median": 0.057809896767139435, "p05": 0.022837381809949875, "p95": 0.14078694581985474 }, { "label": "completed_group_7", "mean": 0.3412485122680664, "median": 0.3336990475654602, "p05": 0.0819513276219368, "p95": 0.6423859596252441 }, { "label": "completed_group_8", "mean": 0.31639498472213745, "median": 0.30713334679603577, "p05": 0.05841828137636185, "p95": 0.6156357526779175 } ], "sources": 9 } ], "mode": "uniform_group_6_attention_only", "optimizer_state_after": "ed03e6fbd4a12d8b063dcb22e0437754285f54d585374cd52fbd534f05d24637", "optimizer_state_before": "ed03e6fbd4a12d8b063dcb22e0437754285f54d585374cd52fbd534f05d24637", "optimizer_state_unchanged": true, "positions": { "attention_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "5e6ecbcd78543fc42d4098a51169dca446d404ce384015643e84d2c78433c949", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.0933690071105957, 0.24537226557731628, 0.07698935270309448, 0.0523640513420105, 0.09380833059549332, 0.03647654876112938, 0.039120711386203766, 0.06371714919805527, 0.05156813934445381, 0.03728010877966881, 0.03983496502041817, 0.040284499526023865, 0.046871740370988846, 0.04314620420336723, 0.04033593460917473, 0.04471700266003609, 0.08397205173969269, 0.0528092160820961, 0.04443390294909477, 0.07560808956623077, 0.03708760067820549, 0.03599191829562187, 0.033304885029792786, 0.04061548039317131, 0.025487789884209633, 0.021565141156315804, 0.02342291921377182, 0.017244853079319, 0.024166326969861984, 0.016819356009364128, 0.021590016782283783, 0.026201345026493073 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.6946603977885388e-05, "first_to_last_ratio": 0.7046201307373134, "imbalance_abs_log_ratio": 0.3500964430356202, "last_quartile_mean": 2.4050695174082648e-05, "mean": 2.103456859003927e-05, "normalized": [ 0.7339764560566742, 0.8481483787913635, 0.7508063661740763, 0.6999799407661862, 0.9491471236867144, 0.883487047828285, 0.8312468663865321, 0.7484472956413805, 0.6765261272520305, 0.6132638632605583, 0.5839073577954953, 0.5686237717151313, 0.692290564201055, 0.6577776130886733, 0.6286425299146282, 0.5959522375125703, 0.6424516536739611, 0.6074612675025246, 0.6210546763995477, 0.6808538715461665, 2.756204145255747, 2.443545320243014, 2.0032723672212382, 1.635820842991328, 2.209968973314971, 1.9375945637433365, 1.4599353583974992, 0.9426010488632537, 0.9953140821258564, 0.7102444918691895, 0.4697227230605014, 0.42173107372051094 ], "peak_layer": 21, "peak_normalized": 2.756204145255747, "population_cv": 0.6083797470397646, "rest_mean": 1.6322192904125484e-05, "spike_contrast": 2.847742185562909, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.648139729397371e-05, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 1.5438878108398058e-05, 1.7840435248217545e-05, 1.579288800712675e-05, 1.472377607569797e-05, 1.9964900275226682e-05, 1.8583768905955367e-05, 1.748491922626272e-05, 1.5743265976198018e-05, 1.4230435226636473e-05, 1.2899740795546677e-05, 1.2282239367777947e-05, 1.1960755728068762e-05, 1.4562033356924076e-05, 1.3836068319506012e-05, 1.3223224414105061e-05, 1.2535598216345534e-05, 1.3513693374989089e-05, 1.2777685697074048e-05, 1.306361718889093e-05, 1.4321467460831627e-05, 5.797556514153257e-05, 5.1398921641521156e-05, 4.213797001284547e-05, 3.4408785722916946e-05, 4.648574395105243e-05, 4.075646575074643e-05, 3.0709110433235765e-05, 1.9827206415357068e-05, 2.0936002329108305e-05, 1.4939686479920056e-05, 9.880414836516138e-06, 8.870931196724996e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 6.757411802027491e-05, "first_to_last_ratio": 0.7038868415923585, "imbalance_abs_log_ratio": 0.3511376721180609, "last_quartile_mean": 9.600139401300112e-05, "mean": 8.399745513543166e-05, "normalized": [ 0.7320878040497206, 0.8478892172223632, 0.756636675968792, 0.7008103297864225, 0.9467328910935654, 0.8784314943281901, 0.8275861684397404, 0.745650686562282, 0.6808778463575155, 0.6174380626069348, 0.5864148214696665, 0.5694394114500967, 0.7006121838662205, 0.6581760408993969, 0.628656413294731, 0.5952106379715426, 0.638807244716704, 0.6051617209817185, 0.6239816421378056, 0.6806185458931985, 2.754032180486398, 2.446159069847505, 2.0065019012148415, 1.6288200988783603, 2.2053085676811155, 1.932346039615062, 1.454690590279931, 0.9391756273523492, 0.9954119082089729, 0.7132529840528576, 0.4751945852826801, 0.42788660800332023 ], "peak_layer": 21, "peak_normalized": 2.754032180486398, "population_cv": 0.6070477733847837, "rest_mean": 6.520435662996197e-05, "spike_contrast": 2.844598070610182, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00018548018706496805, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 6.14935124758631e-05, 7.122053648345172e-05, 6.355555524351075e-05, 5.88662842346821e-05, 7.952315354486927e-05, 7.378601003438234e-05, 6.951513205422089e-05, 6.26327600912191e-05, 5.7192006352124736e-05, 5.186322596273385e-05, 4.925735265715048e-05, 4.783146141562611e-05, 5.8849640481639653e-05, 5.528511246666312e-05, 5.280553887132555e-05, 4.999617885914631e-05, 5.365818287828006e-05, 5.083204450784251e-05, 5.24128699908033e-05, 5.717022577300668e-05, 0.00023133169452194124, 0.00020547113672364503, 0.00016854105342645198, 0.00013681674317922443, 0.00018524030747357756, 0.00016231214976869524, 0.0001221903075929731, 7.888836262281984e-05, 8.361206710105762e-05, 5.991143552819267e-05, 3.991513585788198e-05, 3.594138615881093e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.05992540530860424, "first_to_last_ratio": 0.7038868742138324, "imbalance_abs_log_ratio": 0.3511376257732917, "last_quartile_mean": 0.08513499470427632, "mean": 0.07448978326283395, "normalized": [ 0.7320878377772885, 0.8478892412309806, 0.7566367170905001, 0.700810311447509, 0.9467328978682269, 0.8784315094412039, 0.827586225003894, 0.7456507045141643, 0.6808778249039888, 0.6174380819890257, 0.5864148103375094, 0.5694394097245594, 0.7006121688363213, 0.6581760428148075, 0.6286563945221932, 0.5952106519107266, 0.6388072274905444, 0.6051617419237312, 0.6239815891906979, 0.6806185691502288, 2.7540322632445617, 2.4461590541488416, 2.006501904504882, 1.6288200828488795, 2.205308658518604, 1.932345957036642, 1.4546905357456201, 0.9391755726418203, 0.9954118664325384, 0.713252937273076, 0.4751945920820103, 0.4278866183544228 ], "peak_layer": 21, "peak_normalized": 2.7540322632445617, "population_cv": 0.6070477794331731, "rest_mean": 0.05782387516012898, "spike_contrast": 2.844598127710019, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.1644856870174408, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 0.05453306436538696, 0.06315908581018448, 0.05636170506477356, 0.052203208208084106, 0.07052192836999893, 0.06543417274951935, 0.061646718531847, 0.055543359369039536, 0.05071844160556793, 0.04599282890558243, 0.04368191212415695, 0.04241741821169853, 0.0521884486079216, 0.04902739077806473, 0.04682847857475281, 0.044337112456560135, 0.04758461192250252, 0.04507836699485779, 0.04648025333881378, 0.050699129700660706, 0.20514726638793945, 0.18221385776996613, 0.14946389198303223, 0.12133045494556427, 0.1642729640007019, 0.1439400315284729, 0.10835958272218704, 0.06995898485183716, 0.0741480141878128, 0.05313005670905113, 0.03539714217185974, 0.0318731814622879 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 2.812907695215472e-06, "first_to_last_ratio": 0.4329449129534598, "imbalance_abs_log_ratio": 0.8371447808895189, "last_quartile_mean": 6.49714920086808e-06, "mean": 4.868524769108262e-06, "normalized": [ 0.6314203262683556, 0.5643201437054097, 0.5420304913506234, 0.5427841342249308, 0.5900282532596633, 0.5764224237197104, 0.5792112339444356, 0.5959760867019027, 0.5434613713396567, 0.522896452909898, 0.5265438471713876, 0.5518405518746916, 0.5621625683501856, 0.5680522568225722, 0.5843158018640565, 0.6020775723134717, 0.5702889466990257, 0.5968880987270297, 0.6282166084836535, 0.7073949724227924, 2.8639854592740694, 2.6655367238696606, 2.275392662876568, 1.9325839314612783, 2.5532925531991593, 2.2860943263078313, 1.7562754928987332, 1.1336063815668957, 1.118588913586166, 0.8072551518676266, 0.5443819766775972, 0.4766742842609617 ], "peak_layer": 21, "peak_normalized": 2.8639854592740694, "population_cv": 0.7396734666999918, "rest_mean": 3.553880392077931e-06, "spike_contrast": 3.3674752903191183, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.1967604405072051e-05, "top_five_layers": [ 21, 22, 25, 26, 23 ], "top_five_spike_overlap": 4 }, "values": [ 3.0740854981559096e-06, 2.7474065973365214e-06, 2.6388888727524318e-06, 2.6425580017530592e-06, 2.8725671654683538e-06, 2.806326847348828e-06, 2.819904239004245e-06, 2.9015243399044266e-06, 2.6458551474206615e-06, 2.5457343326706905e-06, 2.563491761975456e-06, 2.686649395400309e-06, 2.736902388278395e-06, 2.765576482488541e-06, 2.844755954356515e-06, 2.9312295737327076e-06, 2.776465862552868e-06, 2.905964493038482e-06, 3.0584881187678548e-06, 3.4439699447830208e-06, 1.3943384146841709e-05, 1.2977231563127134e-05, 1.1077805538661778e-05, 9.408832738699857e-06, 1.2430768038029782e-05, 1.1129906852147542e-05, 8.550470738555305e-06, 5.518990747077623e-06, 5.44587783224415e-06, 3.930141701857792e-06, 2.6503371373109985e-06, 2.3207005597214447e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 5.262683180262684e-05, "first_to_last_ratio": 0.8858843264800002, "imbalance_abs_log_ratio": 0.1211688939206816, "last_quartile_mean": 5.940598589404544e-05, "mean": 5.845883703159416e-05, "normalized": [ 0.8272683192079959, 0.9365635663248325, 0.8472599891964046, 0.7932450106653395, 1.05708063932328, 0.9863879405378823, 0.9224472307528563, 0.831646551379392, 0.7683174523880734, 0.6999110129982006, 0.6632544424363206, 0.6411628962765582, 0.7829569056169384, 0.7299869800019017, 0.6932077512313936, 0.6529982621271514, 0.7116888769793, 0.6729233444528345, 0.6878765687054961, 0.7347363606036672, 2.579411393979688, 2.2748103003494773, 1.871406557230005, 1.5038358135242726, 2.077058881901721, 1.7234791396065356, 1.2870800272088003, 0.863554425885572, 0.8088900620642815, 0.5750620080349625, 0.41525055955879575, 0.3792407294500703 ], "peak_layer": 21, "peak_normalized": 2.579411393979688, "population_cv": 0.5326181203711993, "rest_mean": 4.69694607292897e-05, "spike_contrast": 2.5655280514833483, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012050146906403825, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 4.836114385398105e-05, 5.475041689351201e-05, 4.952983363182284e-05, 4.637218080461025e-05, 6.179570482345298e-05, 5.766309186583385e-05, 5.3925192332826555e-05, 4.861709021497518e-05, 4.491494473768398e-05, 4.0915983845479786e-05, 3.877308336086571e-05, 3.7481637264136225e-05, 4.577075014822185e-05, 4.2674189899116755e-05, 4.05241189582739e-05, 3.817351898760535e-05, 4.160450407653116e-05, 3.933831612812355e-05, 4.021246422780678e-05, 4.295183316571638e-05, 0.00015078939031809568, 0.00013298276462592185, 0.00010940025094896555, 8.791249274509028e-05, 0.00012142244668211788, 0.00010075258614961058, 7.524120155721903e-05, 5.0482387450756505e-05, 4.7286772314691916e-05, 3.3617456210777164e-05, 2.4275064788525924e-05, 2.216997199866455e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 4.037761755171232e-05, "first_to_last_ratio": 1.1619586606079562, "imbalance_abs_log_ratio": 0.1501070817279916, "last_quartile_mean": 3.474961624760908e-05, "mean": 3.967688857642315e-05, "normalized": [ 0.946882436630873, 1.0509736491460855, 0.9627527133183058, 0.9079146843545549, 1.19508595007448, 1.1127115432709858, 1.0369927391400784, 0.9279733658758964, 0.8771618169882459, 0.7978111670720687, 0.7532725979723313, 0.730393528257228, 0.8875347206410501, 0.820008062269565, 0.7758337780298346, 0.7258659616571059, 0.8153826623078453, 0.7681307532333388, 0.7723066423654125, 0.8051705831313694, 2.330696551507456, 2.021091279931092, 1.6616029136559602, 1.3099294799728538, 1.8776697723918978, 1.4571900529760533, 1.09056746158453, 0.8021680983526658, 0.6404155345959941, 0.45310425726977555, 0.35325852669024743, 0.3321467153348195 ], "peak_layer": 21, "peak_normalized": 2.330696551507456, "population_cv": 0.44288659496317, "rest_mean": 3.350347331568763e-05, "spike_contrast": 2.1792764677388634, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 7.3013330984395e-05, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 3.7569348933175206e-05, 4.169936437392607e-05, 3.819903213297948e-05, 3.602322976803407e-05, 4.7417292080353945e-05, 4.414893192006275e-05, 4.114464536542073e-05, 3.6819095839746296e-05, 3.480305167613551e-05, 3.1654664780944586e-05, 2.988751293742098e-05, 2.8979742637602612e-05, 3.5214616218581796e-05, 3.253536851843819e-05, 3.078267036471516e-05, 2.880010288208723e-05, 3.235184703953564e-05, 3.0477038308163173e-05, 3.0642724595963955e-05, 3.1946663511916995e-05, 9.247478737961501e-05, 8.019061351660639e-05, 6.59272336633876e-05, 5.1973926019854844e-05, 7.450009434251115e-05, 5.781676736660302e-05, 4.327032365836203e-05, 3.182753425789997e-05, 2.5409695808775723e-05, 1.7977767129195854e-05, 1.4016199202160351e-05, 1.3178548215364572e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.0001373312506984803, "first_to_last_ratio": 0.6770915195326159, "imbalance_abs_log_ratio": 0.38994883125865454, "last_quartile_mean": 0.00020282524110371014, "mean": 0.00017544319030093902, "normalized": [ 0.7069446339136618, 0.8191512638479912, 0.7315660698813674, 0.670165345284034, 0.9225308545579605, 0.8562082896075618, 0.8137828129002785, 0.741791789476707, 0.6642205597245825, 0.6026671771646809, 0.5758837295412822, 0.5617973933375549, 0.6839732393359865, 0.6471275127666591, 0.617440453464029, 0.5823980483253871, 0.621509922422184, 0.5961281411445828, 0.6108010555125495, 0.6684450505420366, 2.780724051485821, 2.501415801126332, 2.078146571630916, 1.696591410342404, 2.2891246036936526, 1.9639680371043038, 1.5019299058974314, 0.9780185610501557, 0.9858871853016955, 0.6740068018873399, 0.44785334077789735, 0.4078003869509733 ], "peak_layer": 21, "peak_normalized": 2.780724051485821, "population_cv": 0.6339677401953402, "rest_mean": 0.00013420752684059726, "spike_contrast": 2.9664191149252006, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0003981157729867846, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 0.00012402862193994224, 0.00014371451106853783, 0.0001283482852159068, 0.00011757594620576128, 0.00016185175627470016, 0.00015021591389086097, 0.00014277265290729702, 0.000130142318084836, 0.00011653297406155616, 0.00010573385225143284, 0.00010103487875312567, 9.856352698989213e-05, 0.0001199984471895732, 0.00011353411537129432, 0.00010832572297658771, 0.00010217777162324637, 0.00010903968359343708, 0.00010458662291057408, 0.00010716088581830263, 0.00011727413220796734, 0.0004878590989392251, 0.00043885636841878295, 0.0003645966644398868, 0.0002976554096676409, 0.0004016113234683871, 0.000344564818078652, 0.0002635033742990345, 0.00017158669652417302, 0.0001729671930661425, 0.00011824990360764787, 7.857281889300793e-05, 7.154580089263618e-05 ] } } }, "mlp_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "c0b8018c349b1bd09e5d17858eb312a2323da0e92de9651e2bb08ae4adf115c7", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.13583384454250336, 0.0415157787501812, 0.018140379339456558, 0.039518751204013824, 0.022877270355820656, 0.015532293356955051, 0.02314678393304348, 0.05432315170764923, 0.03743875026702881, 0.04006272554397583, 0.045583780854940414, 0.05995568633079529, 0.038651175796985626, 0.04559725895524025, 0.05632574483752251, 0.07952121645212173, 0.042337141931056976, 0.05044656619429588, 0.04418695345520973, 0.06310147047042847, 0.029010163620114326, 0.04586305841803551, 0.06033914536237717, 0.07656749337911606, 0.045884523540735245, 0.051604680716991425, 0.06502768397331238, 0.06792232394218445, 0.060306549072265625, 0.08025261014699936, 0.09663844108581543, 0.12230555713176727 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.6809715930321545e-05, "first_to_last_ratio": 0.8970240690782879, "imbalance_abs_log_ratio": 0.10867258442013854, "last_quartile_mean": 1.8739425740932347e-05, "mean": 1.865125139488555e-05, "normalized": [ 0.8260105724804402, 0.9551422553670907, 0.8468560602635984, 0.7876144775740321, 1.0659393651643034, 0.9849332546614278, 0.9130917974904718, 0.8305304334872169, 0.7085743530050244, 0.6675104234273384, 0.639911427498808, 0.6288088270730129, 0.7599940746007076, 0.7187196342886786, 0.6762709273577673, 0.6495629326193958, 0.7189952438950039, 0.6718577602325617, 0.6851753314913573, 0.8258227366340911, 2.779519758946923, 2.2615422246743138, 1.8494071665010579, 1.5103887231784132, 2.2071689160554295, 1.6583244712439744, 1.0700956477519785, 0.846931155591578, 0.8009366813368265, 0.5329633921293535, 0.47745078010597797, 0.44394919387184517 ], "peak_layer": 21, "peak_normalized": 2.779519758946923, "population_cv": 0.552952275667937, "rest_mean": 1.4777298895717615e-05, "spike_contrast": 2.6777962041398338, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.957059489039239e-05, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 1.540613084216602e-05, 1.781459832272958e-05, 1.579492527525872e-05, 1.468999562348472e-05, 1.9881103071384132e-05, 1.837023773987312e-05, 1.7030304661602713e-05, 1.5490431906073354e-05, 1.3215798389865085e-05, 1.2449904716049787e-05, 1.1935148904740345e-05, 1.1728071513061877e-05, 1.41748405440012e-05, 1.3405020581558347e-05, 1.26132990772021e-05, 1.2115161553083453e-05, 1.3410161045612767e-05, 1.2530987987702247e-05, 1.2779377357219346e-05, 1.5402627468574792e-05, 5.184152178117074e-05, 4.218059257254936e-05, 3.449375799391419e-05, 2.8170639780000784e-05, 4.116646232432686e-05, 3.092982660746202e-05, 1.9958622942795046e-05, 1.579632589709945e-05, 1.4938471394998487e-05, 9.940434210875537e-06, 8.905054528440814e-06, 8.280208021460567e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 6.707350848955684e-05, "first_to_last_ratio": 0.8958270736438665, "imbalance_abs_log_ratio": 0.1100078827990767, "last_quartile_mean": 7.487327684430056e-05, "mean": 7.454373871951248e-05, "normalized": [ 0.8225912570208472, 0.9540759750521303, 0.8530622669128751, 0.7878005225794009, 1.0622324333592001, 0.9790626677708002, 0.9109019599182948, 0.8285712202860098, 0.7147607070045484, 0.6719370506037835, 0.6424346393710746, 0.6293044590365569, 0.7701332966983393, 0.7192323073552366, 0.676560038722961, 0.6497500067333273, 0.7141906359137283, 0.6691526771815973, 0.6886887757920368, 0.8269141537833384, 2.7796605872321534, 2.266128874877332, 1.8415809602307271, 1.5059066454905177, 2.200556204859446, 1.6509873711452387, 1.0622000279825796, 0.8439381036565154, 0.8039441101199606, 0.5386101064329339, 0.4841369123501972, 0.4509930445263113 ], "peak_layer": 21, "peak_normalized": 2.7796605872321534, "population_cv": 0.5511276127059419, "rest_mean": 5.909984072247796e-05, "spike_contrast": 2.672440161812955, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00015794078790349887, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 6.131902773631737e-05, 7.11203902028501e-05, 6.359045073622838e-05, 5.872559631825425e-05, 7.918277697172016e-05, 7.298299169633538e-05, 6.790203769924119e-05, 6.176479655550793e-05, 5.328093538992107e-05, 5.008869993616827e-05, 4.788947990164161e-05, 4.691070716944523e-05, 5.7408615248277783e-05, 5.361426519812085e-05, 5.0433314754627645e-05, 4.843479473493062e-05, 5.323844015947543e-05, 4.9881142331287265e-05, 5.13374361617025e-05, 6.164127262309194e-05, 0.00020720629254356027, 0.00016892571875359863, 0.00013727832993026823, 0.00011225591151742265, 0.00016403768677264452, 0.00012307077122386545, 7.918036135379225e-05, 6.291030149441212e-05, 5.992899968987331e-05, 4.015001104562543e-05, 3.608937549870461e-05, 3.36187076754868e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.05948145827278495, "first_to_last_ratio": 0.8958270749065522, "imbalance_abs_log_ratio": 0.1100078813895572, "last_quartile_mean": 0.06639837077818811, "mean": 0.06610613310476765, "normalized": [ 0.8225913111778765, 0.954075997600901, 0.8530622602991957, 0.7878004969397062, 1.0622323867651973, 0.9790626560094096, 0.9109019727686168, 0.8285712280396718, 0.7147606997017303, 0.671937078893822, 0.6424346717420987, 0.6293044953394733, 0.7701332702947965, 0.7192323239111584, 0.67656001137045, 0.6497499897241311, 0.7141906310123469, 0.6691526686583105, 0.6886887355509487, 0.826914163012851, 2.7796605647507637, 2.266128758850027, 1.8415810913002935, 1.5059066590587888, 2.200556207746177, 1.6509873372053603, 1.062200040044388, 0.8439381181977654, 0.8039441028694261, 0.5386100945671475, 0.48413692060145963, 0.45099305599571077 ], "peak_layer": 21, "peak_normalized": 2.7796605647507637, "population_cv": 0.5511276068224332, "rest_mean": 0.05241032986453286, "spike_contrast": 2.6724401652129144, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.14006347060203553, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 0.05437833070755005, 0.06307027488946915, 0.056392647325992584, 0.05207844451069832, 0.07022007554769516, 0.06472204625606537, 0.060216207057237625, 0.054773639887571335, 0.047250065952539444, 0.04441916197538376, 0.04246887192130089, 0.04160088673233986, 0.05091053247451782, 0.0475456677377224, 0.04472476616501808, 0.042952459305524826, 0.0472123809158802, 0.044235095381736755, 0.045526549220085144, 0.05466409772634506, 0.1837526112794876, 0.1498050093650818, 0.12173980474472046, 0.09954966604709625, 0.1454702615737915, 0.10914038866758347, 0.07021793723106384, 0.055789485573768616, 0.0531456358730793, 0.035605430603027344, 0.03200441971421242, 0.029813406988978386 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 2.7827112774048146e-06, "first_to_last_ratio": 0.5411014540663938, "imbalance_abs_log_ratio": 0.6141484870955246, "last_quartile_mean": 5.142679356140434e-06, "mean": 4.259176506593576e-06, "normalized": [ 0.7177808848549336, 0.6434057616912334, 0.6207141259471038, 0.6213679783461574, 0.6716677949353012, 0.6517969920427383, 0.6503947968179228, 0.6496309191253968, 0.5884857369270009, 0.5854578792874255, 0.5928438275907121, 0.6246858735497479, 0.6267734713669417, 0.6293729724054389, 0.6380531192480996, 0.6749306507999445, 0.6479976761227316, 0.6730603809641802, 0.7051714310250148, 0.8140514151105848, 2.877200608467633, 2.4671847593668157, 2.1394603559642555, 1.8290295157196803, 2.6371216812014318, 2.034816852829379, 1.370051183181964, 1.0263308851482142, 0.9143507609158581, 0.6183689482943914, 0.5526342005790595, 0.5058065601727082 ], "peak_layer": 21, "peak_normalized": 2.877200608467633, "population_cv": 0.6761489217371661, "rest_mean": 3.1628334100888524e-06, "spike_contrast": 3.2184525417142082, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.0179429227719084e-05, "top_five_layers": [ 21, 25, 22, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 3.057155481656082e-06, 2.7403787044022465e-06, 2.643731022544671e-06, 2.6465158953214996e-06, 2.8607516924239462e-06, 2.776118435576791e-06, 2.770146238617599e-06, 2.766892748695682e-06, 2.50646462518489e-06, 2.4935684450611006e-06, 2.5250265025533736e-06, 2.660647396623972e-06, 2.6695388442021795e-06, 2.680610577954212e-06, 2.7175808554602554e-06, 2.8746487714670366e-06, 2.7599364784691716e-06, 2.8666829621215584e-06, 3.003449592142715e-06, 3.4671886623982573e-06, 1.2254505236342084e-05, 1.0508175364520866e-05, 9.112339284911286e-06, 7.790159543219488e-06, 1.1231966709601693e-05, 8.666644134791568e-06, 5.835289812239353e-06, 4.371324394014664e-06, 3.8943812796787824e-06, 2.6337424969824497e-06, 2.353766603846452e-06, 2.1543194179685088e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 5.229475937085226e-05, "first_to_last_ratio": 1.105494014565858, "imbalance_abs_log_ratio": 0.10029230706810453, "last_quartile_mean": 4.730442560685333e-05, "mean": 5.283686641632812e-05, "normalized": [ 0.9134584607638119, 1.0349185947941697, 0.938001195254325, 0.8761726124335265, 1.1651492977446205, 1.0805316673284877, 0.998238240086036, 0.9114498110664012, 0.7967944228487845, 0.7497875582524679, 0.7143608108195802, 0.6965399230832376, 0.8463876829900754, 0.7846831044919396, 0.7343548381627399, 0.7009891375896985, 0.7821041450751427, 0.732734588456523, 0.7471560639671582, 0.866774379212827, 2.584550834361983, 2.1052367400709486, 1.696435318104223, 1.3808543006184604, 1.9978399080440128, 1.467434518049127, 0.9715154118317991, 0.788461139577348, 0.6417856457216125, 0.4665400998634145, 0.42395088065532394, 0.40480866868019344 ], "peak_layer": 21, "peak_normalized": 2.584550834361983, "population_cv": 0.4840819706670145, "rest_mean": 4.351230018073693e-05, "spike_contrast": 2.3715023949527487, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00010318952408852055, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 4.826428266824223e-05, 5.468185554491356e-05, 4.956104385200888e-05, 4.629421528079547e-05, 6.156283780001104e-05, 5.709190736524761e-05, 5.2743780543096364e-05, 4.8158151912502944e-05, 4.210012048133649e-05, 3.9616425056010485e-05, 3.774458673433401e-05, 3.6802986869588494e-05, 4.472047294257209e-05, 4.146019637119025e-05, 3.880100848618895e-05, 3.7038069422123954e-05, 4.132393223699182e-05, 3.8715399568900466e-05, 3.9477385143982247e-05, 4.579764208756387e-05, 0.0001365595671813935, 0.0001112341124098748, 8.963432628661394e-05, 7.296001422218978e-05, 0.00010555960034253076, 7.753464160487056e-05, 5.133183003636077e-05, 4.165981590631418e-05, 3.390994243090972e-05, 2.4650516934343614e-05, 2.2400236048270017e-05, 2.138882155122701e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 4.018822119178367e-05, "first_to_last_ratio": 1.4151828379848426, "imbalance_abs_log_ratio": 0.3472587368713388, "last_quartile_mean": 2.8397900337040483e-05, "mean": 3.662407780780086e-05, "normalized": [ 1.02332558389414, 1.1363645290553046, 1.0453075751491576, 0.9835286199165135, 1.2907162594766834, 1.1939578904881842, 1.0995068279709794, 1.0058282780432033, 0.8996084426610936, 0.842707630768332, 0.7976072789612582, 0.77534492036875, 0.9450217926897555, 0.8634110078982206, 0.8033675985010532, 0.761764810604239, 0.8781481482197162, 0.8182124654399122, 0.8211917595243591, 0.91794461553244, 2.320257107961356, 1.8762309371076253, 1.4966879625479523, 1.2008474273504208, 1.7285164377779867, 1.2333783824696765, 0.8916676648661159, 0.7609949303387935, 0.4924089313352787, 0.3920329545295731, 0.36076390076147585, 0.343347327790449 ], "peak_layer": 21, "peak_normalized": 2.320257107961356, "population_cv": 0.4108882703851846, "rest_mean": 3.171029328330877e-05, "spike_contrast": 1.9917354177642579, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.315851424005814e-05, "top_five_layers": [ 21, 22, 25, 23, 5 ], "top_five_spike_overlap": 4 }, "values": [ 3.747835580725223e-05, 4.1618302930146456e-05, 3.8283425965346396e-05, 3.602082870202139e-05, 4.727129271486774e-05, 4.372760668047704e-05, 4.026842361781746e-05, 3.683753311634064e-05, 3.294732960057445e-05, 3.086338983848691e-05, 2.9211631044745445e-05, 2.839629269146826e-05, 3.461055166553706e-05, 3.162163193337619e-05, 2.9422597435768694e-05, 2.7898933694814332e-05, 3.216136610717513e-05, 2.9966276997583918e-05, 3.007539089594502e-05, 3.361887502251193e-05, 8.49772768560797e-05, 6.871522782603279e-05, 5.481481639435515e-05, 4.3979929614579305e-05, 6.330532050924376e-05, 4.5171345846029e-05, 3.265650593675673e-05, 2.7870737540069968e-05, 1.8034023014479317e-05, 1.435784542991314e-05, 1.3212645171734039e-05, 1.2574779248097911e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00013583853251475375, "first_to_last_ratio": 0.8633950542640468, "imbalance_abs_log_ratio": 0.1468829240865166, "last_quartile_mean": 0.00015733068175904918, "mean": 0.00015515380209762952, "normalized": [ 0.7906139891060681, 0.9261413817982224, 0.8265944646589645, 0.7584202952459309, 1.0376523719106512, 0.9619345582046064, 0.8978965391977819, 0.8048174920848862, 0.6997926610992554, 0.6538097358327316, 0.6296879598974191, 0.6206382316310994, 0.7479009674169154, 0.706351132515468, 0.6592007998995999, 0.6381980190155001, 0.6959197799490607, 0.6592141650114567, 0.6805836658068553, 0.8003334736093659, 2.8518591979955707, 2.3506194827532303, 1.9168808042376337, 1.5726951280612045, 2.2230082692614217, 1.7092329232097327, 1.1100251560421506, 0.8952348654477182, 0.7625666687903322, 0.5112650966742065, 0.4612544894355957, 0.43965623419936395 ], "peak_layer": 21, "peak_normalized": 2.8518591979955707, "population_cv": 0.5801662384114423, "rest_mean": 0.00012116326521478456, "spike_contrast": 2.795424014568924, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00033870270126499233, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 0.0001226667664013803, 0.00014369435666594654, 0.000128249273984693, 0.00011767179239541292, 0.00016099571075756103, 0.00014924780407454818, 0.0001393120619468391, 0.00012487049389164895, 0.0001085754920495674, 0.00010144106636289507, 9.769848111318424e-05, 9.629438136471435e-05, 0.00011603967868722975, 0.00010959306382574141, 0.0001022775104502216, 9.901884914143011e-05, 0.00010797459981404245, 0.00010227958409814164, 0.00010559514339547604, 0.00012417478137649596, 0.00044247679761610925, 0.00036470755003392696, 0.00029741134494543076, 0.00024400962865911424, 0.00034490818507038057, 0.0002651939867064357, 0.00017222462338395417, 0.00013889909314457327, 0.00011831511801574379, 7.932472362881526e-05, 7.156538777053356e-05, 6.821433635195717e-05 ] } } }, "post_attention_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "90a633f12d984935028fe5d7212be48e7da7b4a0806751939a551938d16dc540", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.0933690071105957, 0.35791492462158203, 0.3940892517566681, 0.4057151973247528, 0.09380833059549332, 0.11286329478025436, 0.13231655955314636, 0.1621972620487213, 0.05156813934445381, 0.08533531427383423, 0.12003090232610703, 0.16063299775123596, 0.046871740370988846, 0.079384446144104, 0.11459959298372269, 0.15625107288360596, 0.08397205173969269, 0.11654024571180344, 0.15294702351093292, 0.21572710573673248, 0.03708760067820549, 0.06553087383508682, 0.09390194714069366, 0.12852425873279572, 0.025487789884209633, 0.06059965491294861, 0.08860157430171967, 0.12100546807050705, 0.024166326969861984, 0.06877554953098297, 0.12682415544986725, 0.2025238573551178 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.6946694813668728e-05, "first_to_last_ratio": 0.7046180288876429, "imbalance_abs_log_ratio": 0.3500994259944, "last_quartile_mean": 2.4050895831351227e-05, "mean": 2.1034728661106783e-05, "normalized": [ 0.7339795181615647, 0.8481345740915083, 0.7507936481341849, 0.6999957572940897, 0.9491411979553076, 0.8834753955215293, 0.8312562792723062, 0.7484486045902403, 0.6765249136285064, 0.6132726001242113, 0.5839059842312944, 0.5686248925338807, 0.6922773834643536, 0.6577698402845521, 0.6286286661100164, 0.5959549663520993, 0.6424561473106406, 0.6074619630605702, 0.6210560900314085, 0.6808644721373937, 2.756175907018911, 2.443573594976593, 2.0032773579000236, 1.6358312241860395, 2.2099492156209624, 1.9376351632871578, 1.4599235567084665, 0.9426107385426807, 0.9952914611949416, 0.7102564253619122, 0.469730649788516, 0.4217218111241371 ], "peak_layer": 21, "peak_normalized": 2.756175907018911, "population_cv": 0.6083802772882506, "rest_mean": 1.6322320502863646e-05, "spike_contrast": 2.8477404733882543, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.648173271561973e-05, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 1.5439060007338412e-05, 1.7840280634118244e-05, 1.579274066898506e-05, 1.4724220818607137e-05, 1.9964927560067736e-05, 1.8583665223559365e-05, 1.7485250282334164e-05, 1.5743413314339705e-05, 1.4230517990654334e-05, 1.2900022738904227e-05, 1.2282303941901773e-05, 1.1960870324401185e-05, 1.4561866919393651e-05, 1.3836010111845098e-05, 1.3223033420217689e-05, 1.2535751011455432e-05, 1.3513890735339373e-05, 1.2777797564922366e-05, 1.3063746337138582e-05, 1.4321799426397774e-05, 5.797541234642267e-05, 5.139990753377788e-05, 4.21383956563659e-05, 3.440926593611948e-05, 4.648568210541271e-05, 4.07576299039647e-05, 3.0709095881320536e-05, 1.982756111829076e-05, 2.0935685824952088e-05, 1.4940051187295467e-05, 9.88065676210681e-06, 8.870803867466748e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 6.75743922329275e-05, "first_to_last_ratio": 0.7038875973658615, "imbalance_abs_log_ratio": 0.3511365984041447, "last_quartile_mean": 9.600168050383218e-05, "mean": 8.399783018830931e-05, "normalized": [ 0.7320908585722918, 0.8478896757893544, 0.7566309588018466, 0.700820843419827, 0.9467154109228653, 0.8784271390008321, 0.8275969389125677, 0.7456508220404108, 0.680875802356109, 0.6174456136000083, 0.5864149316641625, 0.5694416763390499, 0.700603685123025, 0.6581685112219008, 0.628645810454394, 0.5952075472339597, 0.6388095030488945, 0.6051612710564426, 0.623987777981108, 0.6806336539641226, 2.7540252541369785, 2.4461928439952922, 2.006498832335637, 1.628811266972814, 2.2052855545523546, 1.9323673824210337, 1.454677511862689, 0.939183993917662, 0.9953936909602911, 0.7132599772982093, 0.47520073580888234, 0.42788452423498397 ], "peak_layer": 21, "peak_normalized": 2.7540252541369785, "population_cv": 0.6070480826249149, "rest_mean": 6.520467286463827e-05, "spike_contrast": 2.8445948976875046, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00018548087973613292, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 6.149404362076893e-05, 7.122089300537482e-05, 6.355535879265517e-05, 5.886743019800633e-05, 7.95220403233543e-05, 7.378597365459427e-05, 6.951634713914245e-05, 6.263305112952366e-05, 5.71920900256373e-05, 5.186409180168994e-05, 4.9257581849815324e-05, 4.7831865231273696e-05, 5.8849189372267574e-05, 5.528472684090957e-05, 5.2804884035140276e-05, 4.999614247935824e-05, 5.3658612159779295e-05, 5.083223368274048e-05, 5.241361941443756e-05, 5.717175008612685e-05, 0.00023133214563131332, 0.00020547489111777395, 0.00016854154819156975, 0.00013681661221198738, 0.0001852392015280202, 0.00016231466725002974, 0.00012218975462019444, 7.88894176366739e-05, 8.361091022379696e-05, 5.9912290453212336e-05, 3.991583071183413e-05, 3.59413716068957e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.05992564558982849, "first_to_last_ratio": 0.7038875790469808, "imbalance_abs_log_ratio": 0.3511366244294376, "last_quartile_mean": 0.08513525081798434, "mean": 0.07449011504650116, "normalized": [ 0.732090828328446, 0.8478896655635677, 0.7566309964878604, 0.7008208428695083, 0.9467153782586613, 0.8784270967465748, 0.8275969919264997, 0.745650784055122, 0.6808757924418882, 0.617445634050531, 0.586414948988257, 0.5694416744166322, 0.7006036971386174, 0.6581685102882667, 0.6286457928039645, 0.5952075507101796, 0.6388094832771781, 0.6051612969691417, 0.6239877618226398, 0.6806336914554205, 2.7540251977066372, 2.4461927681969824, 2.0064987686406757, 1.6288113276656064, 2.2052856331543422, 1.9323673565652926, 1.4546775550931026, 0.9391839921487465, 0.995393729919521, 0.7132600125586448, 0.4752007272731637, 0.4278845124783279 ], "peak_layer": 21, "peak_normalized": 2.7540251977066372, "population_cv": 0.6070480747332982, "rest_mean": 0.05782415552271737, "spike_contrast": 2.8445948754117456, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.16448629647493362, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 0.05453353002667427, 0.06315939873456955, 0.05636152997612953, 0.05220422521233559, 0.07052093744277954, 0.06543413549661636, 0.061647795140743256, 0.05554361268877983, 0.0507185161113739, 0.04599359631538391, 0.04368211701512337, 0.042417775839567184, 0.052188050001859665, 0.04902704805135727, 0.04682789742946625, 0.04433707892894745, 0.04758499190211296, 0.04507853463292122, 0.046480920165777206, 0.05070048198103905, 0.2051476538181305, 0.18221718072891235, 0.14946432411670685, 0.12133034318685532, 0.1642719805240631, 0.14394226670265198, 0.10835909843444824, 0.06995992362499237, 0.07414699345827103, 0.05313082039356232, 0.03539775684475899, 0.03187316656112671 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 2.812894507542296e-06, "first_to_last_ratio": 0.43294397786971783, "imbalance_abs_log_ratio": 0.837146940713376, "last_quartile_mean": 6.497132773120029e-06, "mean": 4.868523120649115e-06, "normalized": [ 0.6314270317539187, 0.5642822719972522, 0.542032869911345, 0.5427943124089073, 0.5899768931461296, 0.5764146327143594, 0.5792601877888178, 0.5959847884067344, 0.5434318990733282, 0.5228708500130205, 0.526553085800422, 0.5518437277047022, 0.5621267508348082, 0.5680363366954272, 0.5843255270824329, 0.6020799712058539, 0.5702826481065992, 0.5969250092335819, 0.6282383511869684, 0.7073501904414786, 2.8638409030724836, 2.665701646651509, 2.275485157618661, 1.9325892561054385, 2.553326483316483, 2.2860777269260146, 1.7562105168302045, 1.1336314244806134, 1.118537825846404, 0.8072612163479962, 0.5443997679594592, 0.47670073933864476 ], "peak_layer": 21, "peak_normalized": 2.8638409030724836, "population_cv": 0.7396786801102861, "rest_mean": 3.5538517598373196e-06, "spike_contrast": 3.367542958398649, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.1967748469032813e-05, "top_five_layers": [ 21, 22, 25, 26, 23 ], "top_five_spike_overlap": 4 }, "values": [ 3.074117103096796e-06, 2.747221287791035e-06, 2.6388995593151776e-06, 2.642606659719604e-06, 2.8723161449306644e-06, 2.806287966450327e-06, 2.8201416171214078e-06, 2.9015657219133573e-06, 2.645710765136755e-06, 2.545608822401846e-06, 2.563535872468492e-06, 2.6866639473155374e-06, 2.736727083174628e-06, 2.765498038570513e-06, 2.8448023385863053e-06, 2.9312402602954535e-06, 2.7764342576119816e-06, 2.9061432087473804e-06, 3.058592938032234e-06, 3.4437507565598935e-06, 1.3942675650469027e-05, 1.297803009947529e-05, 1.1078252100560348e-05, 9.408855476067401e-06, 1.2430929018591996e-05, 1.1129822269140277e-05, 8.550151505914982e-06, 5.519110800378257e-06, 5.445627266453812e-06, 3.930169896193547e-06, 2.6504228571866406e-06, 2.3208285711007193e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 5.262699505692581e-05, "first_to_last_ratio": 0.8858859658370462, "imbalance_abs_log_ratio": 0.12116704339067429, "last_quartile_mean": 5.940606024523731e-05, "mean": 5.8459032743485295e-05, "normalized": [ 0.8272728929280775, 0.9365662183598569, 0.8472541033683268, 0.7932478313526844, 1.0570721218789738, 0.9863863807371405, 0.9224526059910532, 0.8316453229353727, 0.7683164981914334, 0.6999158886285397, 0.6632555202183601, 0.641162430005156, 0.782954035472086, 0.7299843494235023, 0.6932022566841178, 0.6529964493764613, 0.711691348391572, 0.6729223362335496, 0.6878797421502519, 0.7347432977378998, 2.579421178955903, 2.2748196115334656, 1.871407013022609, 1.5038317746122511, 2.077049687901519, 1.7234827043500753, 1.2870687483616894, 0.86355557986896, 0.8088879141054698, 0.5750619497529701, 0.4152522809261076, 0.3792399265445636 ], "peak_layer": 21, "peak_normalized": 2.579421178955903, "population_cv": 0.5326188581607553, "rest_mean": 4.696960429471262e-05, "spike_contrast": 2.5655303717434697, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012050194636685774, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 4.8361573135480285e-05, 5.475075522554107e-05, 4.952965537086129e-05, 4.6372500946745276e-05, 6.179541378514841e-05, 5.766319372924045e-05, 5.392568709794432e-05, 4.8617181164445356e-05, 4.4915039325132966e-05, 4.091640585102141e-05, 3.877327617374249e-05, 3.7481735489564016e-05, 4.577073559630662e-05, 4.267417898518033e-05, 4.052393342135474e-05, 3.817354081547819e-05, 4.160478783887811e-05, 3.9338388887699693e-05, 4.02127843699418e-05, 4.295238250051625e-05, 0.00015079046715982258, 0.00013298375415615737, 0.00010940064385067672, 8.791255095275119e-05, 0.00012142231571488082, 0.00010075313184643164, 7.524079410359263e-05, 5.0482623919378966e-05, 4.728680505650118e-05, 3.3617565350141376e-05, 2.427524668746628e-05, 2.2169999283505604e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 4.037808957946254e-05, "first_to_last_ratio": 1.1619132592825117, "imbalance_abs_log_ratio": 0.1500680078659068, "last_quartile_mean": 3.475138032627001e-05, "mean": 3.967591527498371e-05, "normalized": [ 0.946932805822147, 1.0513569394477453, 0.9628715075684395, 0.9078279344432144, 1.1951496517159068, 1.1125783778700356, 1.0370626487118655, 0.9278021092005068, 0.8771070468463767, 0.7978041476222385, 0.7534492460646955, 0.7304564667086636, 0.8874195046092345, 0.8197357711500388, 0.775778906182002, 0.7258233428129827, 0.8151978239371286, 0.7681149826020242, 0.7722520507618774, 0.8050421601314364, 2.331105458445079, 2.0210222099800808, 1.6615406126944225, 1.3095202988452708, 1.877750493773097, 1.4571729849232538, 1.090841233864544, 0.8022280295236202, 0.6405045070218112, 0.4531065700345776, 0.35333596185756616, 0.33210821482811714 ], "peak_layer": 21, "peak_normalized": 2.331105458445079, "population_cv": 0.44291946725238474, "rest_mean": 3.350272628368758e-05, "spike_contrast": 2.1792595387537697, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 7.30111358279828e-05, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 3.7570425774902105e-05, 4.1713548853294924e-05, 3.8202808354981244e-05, 3.601890421123244e-05, 4.7418656322406605e-05, 4.414256545715034e-05, 4.114640978514217e-05, 3.681139787659049e-05, 3.480002487776801e-05, 3.165360976709053e-05, 2.9893788450863212e-05, 2.8981528885196894e-05, 3.5209181078244e-05, 3.2523767004022375e-05, 3.077973815379664e-05, 2.8797705454053357e-05, 3.23437197948806e-05, 3.0475664971163496e-05, 3.063980693696067e-05, 3.1940784538164735e-05, 9.248874266631901e-05, 8.018590597203001e-05, 6.592314457520843e-05, 5.195641642785631e-05, 7.450146949850023e-05, 5.781467189081013e-05, 4.3280124373268336e-05, 3.182913133059628e-05, 2.541260255384259e-05, 1.7977417883230373e-05, 1.401892768626567e-05, 1.3176697393646464e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00013734229378314922, "first_to_last_ratio": 0.6772042517137701, "imbalance_abs_log_ratio": 0.3897823503704941, "last_quartile_mean": 0.0002028077842624043, "mean": 0.00017543686226417776, "normalized": [ 0.7069310656110243, 0.8190173227520255, 0.731703689164984, 0.6700848394775869, 0.922590175674449, 0.8563731321626127, 0.8138735467666921, 0.7422967339670568, 0.6641711518818515, 0.6028175658585425, 0.5759526938171036, 0.5616621738048295, 0.6838712092239675, 0.6472671460884367, 0.6174174357254809, 0.5825905064245618, 0.6215096959291867, 0.5959259362366661, 0.6108943799282998, 0.668287549585948, 2.7810466499186837, 2.501254201317018, 2.077779922296534, 1.6965548954204952, 2.2886849401438205, 1.9639346966562667, 1.501820343900281, 0.9780539212888347, 0.9858349058220159, 0.6740304913241083, 0.4479093508292885, 0.40785773100134803 ], "peak_layer": 21, "peak_normalized": 2.7810466499186837, "population_cv": 0.633919587741048, "rest_mean": 0.00013420711642269184, "spike_contrast": 2.9661429320516612, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0003980774898082018, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 0.00012402176798786968, 0.00014368582924362272, 0.00012836779933422804, 0.00011755758168874308, 0.00016185632557608187, 0.0001502394152339548, 0.000142783421324566, 0.00013022620987612754, 0.00011652010289253667, 0.00010575642227195203, 0.00010104333341587335, 9.853624942479655e-05, 0.00011997621913906187, 0.00011355451715644449, 0.00010831777763087302, 0.0001022078504320234, 0.00010903571092057973, 0.00010454737639520317, 0.00010717339318944141, 0.00011724227078957483, 0.0004878980980720371, 0.00043881218880414963, 0.000364519190043211, 0.0002976382675115019, 0.00040151970461010933, 0.0003445465408731252, 0.0002634746488183737, 0.00017158671107608825, 0.00017295178258791566, 0.00011824979446828365, 7.857981108827516e-05, 7.155328057706356e-05 ] } } }, "post_mlp_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "89f2226456bfe96e245d2aad0f699d32c676f9f53099789cd870253d002402f5", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.16915453970432281, 0.374446302652359, 0.39966297149658203, 0.4194087088108063, 0.09892481565475464, 0.11664740741252899, 0.13951337337493896, 0.18593810498714447, 0.06607726216316223, 0.10425819456577301, 0.14551182091236115, 0.19865906238555908, 0.06484340131282806, 0.10430748760700226, 0.14720691740512848, 0.20439592003822327, 0.10052042454481125, 0.14085541665554047, 0.1745898574590683, 0.2420409768819809, 0.049456026405096054, 0.08509506285190582, 0.11783231794834137, 0.15649692714214325, 0.05533855780959129, 0.08507220447063446, 0.11942370235919952, 0.1559232622385025, 0.06628894805908203, 0.12384890764951706, 0.19928614795207977, 0.30097496509552 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.680974764894927e-05, "first_to_last_ratio": 0.8970300173599999, "imbalance_abs_log_ratio": 0.10866595331380016, "last_quartile_mean": 1.873933683782525e-05, "mean": 1.8651231073363306e-05, "normalized": [ 0.8260078639827807, 0.9551330557629407, 0.8468556175878655, 0.7876136290086277, 1.065946670732628, 0.9849382288595823, 0.9131007895267456, 0.8305438217877184, 0.7085763928795813, 0.6675024708568932, 0.6399139289571883, 0.6288115114875238, 0.7600116772143157, 0.7187212951110417, 0.6762638133076667, 0.6495449152640493, 0.7189961735688618, 0.6718442533879192, 0.6851708115955005, 0.8258452872968888, 2.779501331552247, 2.2615630237297197, 1.8494175688094532, 1.5103850048705498, 2.2071611781283633, 1.6583054074055965, 1.070091644774074, 0.8469281773085036, 0.8009316536469137, 0.5329766025042728, 0.4774465215155496, 0.44394967757843706 ], "peak_layer": 21, "peak_normalized": 2.779501331552247, "population_cv": 0.55295112786718, "rest_mean": 1.4777281884798194e-05, "spike_contrast": 2.6777967017278232, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.9570556691614914e-05, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 1.540606353955809e-05, 1.7814407328842208e-05, 1.579489980940707e-05, 1.4689963791170157e-05, 1.9881217667716555e-05, 1.837031049944926e-05, 1.7030453818733804e-05, 1.5490664736717008e-05, 1.3215822036727332e-05, 1.2449742825992871e-05, 1.193518255604431e-05, 1.172810880234465e-05, 1.417515341017861e-05, 1.3405036952462979e-05, 1.2613152648555115e-05, 1.2114812307117973e-05, 1.3410163774096873e-05, 1.2530722415249329e-05, 1.2779279131791554e-05, 1.540303128422238e-05, 5.184112160350196e-05, 4.2180934542557225e-05, 3.449391442700289e-05, 2.817053973558359e-05, 4.116627314942889e-05, 3.092943734372966e-05, 1.9958526536356658e-05, 1.579625313752331e-05, 1.4938361346139573e-05, 9.940669770003296e-06, 8.90496539796004e-06, 8.280208021460567e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 6.707344118694891e-05, "first_to_last_ratio": 0.8958282422863163, "imbalance_abs_log_ratio": 0.11000657825970529, "last_quartile_mean": 7.487310404030723e-05, "mean": 7.454362673797732e-05, "normalized": [ 0.822597177862327, 0.9540673548015464, 0.8530621819131958, 0.7878019012489349, 1.062226806178672, 0.979072337509337, 0.9109037187276798, 0.8285704152482716, 0.7147601214242633, 0.6719303002736763, 0.6424466828152381, 0.6293048187552195, 0.7701420669370753, 0.7192365600240997, 0.6765539785845557, 0.6497448335838127, 0.7141918552010665, 0.6691466547189963, 0.6886840515642485, 0.8269385287856083, 2.7796448511492065, 2.2661219328094524, 1.841595829936639, 1.5058956331943163, 2.200551897274131, 1.650978919354254, 1.062190691698668, 0.8439439589591267, 0.8039445369738193, 0.5386226771558794, 0.48413300331592085, 0.45099372202076227 ], "peak_layer": 21, "peak_normalized": 2.7796448511492065, "population_cv": 0.5511247569705954, "rest_mean": 5.909981579558613e-05, "spike_contrast": 2.672431439941738, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00015794020582688973, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 6.131937698228285e-05, 7.111964077921584e-05, 6.359034887282178e-05, 5.8725610870169476e-05, 7.918223855085671e-05, 7.298360287677497e-05, 6.790206680307165e-05, 6.176464376039803e-05, 5.328081169864163e-05, 5.008812149753794e-05, 4.789030572283082e-05, 4.6910663513699546e-05, 5.740918277297169e-05, 5.361450166674331e-05, 5.043278724770062e-05, 4.8434336349600926e-05, 5.323845107341185e-05, 4.9880618462339044e-05, 5.133700688020326e-05, 6.16429970250465e-05, 0.00020720480824820697, 0.00016892494750209153, 0.0001372792321490124, 0.00011225492198718712, 0.0001640371192479506, 0.00012306995631661266, 7.917954644653946e-05, 6.291064346441999e-05, 5.9928941482212394e-05, 4.015088779851794e-05, 3.608902989071794e-05, 3.36187076754868e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.05948139913380146, "first_to_last_ratio": 0.8958282512035028, "imbalance_abs_log_ratio": 0.11000656830558007, "last_quartile_mean": 0.06639821757562459, "mean": 0.06610603345325217, "normalized": [ 0.8225971721599233, 0.9540674049426457, 0.8530622501226689, 0.7878019099222999, 1.062226774810105, 0.9790723594725806, 0.9109037403802558, 0.8285703919985482, 0.7147600865697679, 0.6719303150556425, 0.6424466854183949, 0.6293048240984722, 0.7701420952748721, 0.719236563900054, 0.6765539870934505, 0.649744826684488, 0.7141918203252615, 0.6691466895683721, 0.6886840820355602, 0.8269385143797051, 2.77964491860255, 2.2661218059256747, 1.841595701575012, 1.505895629768547, 2.2005520863438757, 1.6509788934537306, 1.0621905960190525, 0.8439439550057914, 0.803944525829065, 0.5386226843254658, 0.4841329730931415, 0.4509937358450249 ], "peak_layer": 21, "peak_normalized": 2.77964491860255, "population_cv": 0.5511247588690867, "rest_mean": 0.05241030751279107, "spike_contrast": 2.672431439131679, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.14006295353174208, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 0.05437863618135452, 0.06306961178779602, 0.05639256164431572, 0.05207845941185951, 0.07021959871053696, 0.06472259014844894, 0.060216233134269714, 0.05477350205183029, 0.04724995419383049, 0.04441864788532257, 0.04246960207819939, 0.041600845754146576, 0.05091103911399841, 0.04754587635397911, 0.044724300503730774, 0.042952053248882294, 0.0472123883664608, 0.044234633445739746, 0.045526172965765, 0.05466562509536743, 0.18375129997730255, 0.14980432391166687, 0.12174058705568314, 0.09954878687858582, 0.1454697698354721, 0.10913966596126556, 0.07021720707416534, 0.05578978732228279, 0.05314558371901512, 0.03560620918869972, 0.032004110515117645, 0.029813406988978386 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 2.782681661983588e-06, "first_to_last_ratio": 0.54109561457137, "imbalance_abs_log_ratio": 0.6141592790209383, "last_quartile_mean": 5.1426801235265884e-06, "mean": 4.259182233568026e-06, "normalized": [ 0.7177799197136077, 0.6433569040253319, 0.6207332036861026, 0.621337461144743, 0.6716359288754298, 0.651832523751323, 0.6503695790215253, 0.6496510790536977, 0.5884761906048849, 0.5854238336199322, 0.5928535471590092, 0.6246502269894224, 0.6267815437823155, 0.6293183680984733, 0.6380522079258376, 0.674927180827024, 0.648034334011298, 0.6730285664161623, 0.705168988076789, 0.8140417790261263, 2.877408141757337, 2.4670821470538318, 2.1395127853933613, 1.8290740346011667, 2.6370745736496417, 2.0347846486161165, 1.3700990952003282, 1.0263701839996726, 0.9143309537092456, 0.6183935811591681, 0.5526106089951529, 0.5058058800559422 ], "peak_layer": 21, "peak_normalized": 2.877408141757337, "population_cv": 0.6761634910964767, "rest_mean": 3.162813249622963e-06, "spike_contrast": 3.2185190662410648, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.0179574746871368e-05, "top_five_layers": [ 21, 25, 22, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 3.057155481656082e-06, 2.740174295468023e-06, 2.643815832925611e-06, 2.646389475557953e-06, 2.860619815692189e-06, 2.7762735044234432e-06, 2.770042556221597e-06, 2.7669823339238064e-06, 2.506427335902117e-06, 2.4934267912612995e-06, 2.525071295167436e-06, 2.660499148987583e-06, 2.6695768156059785e-06, 2.680381612663041e-06, 2.71758062808658e-06, 2.8746378575306153e-06, 2.760096322163008e-06, 2.8665513127634767e-06, 3.0034432256798027e-06, 3.4671522826101864e-06, 1.2255405636096839e-05, 1.0507752449484542e-05, 9.112574844039045e-06, 7.790359632053878e-06, 1.1231781172682531e-05, 8.666518624522723e-06, 5.835501724504866e-06, 4.371497652755352e-06, 3.894302153639728e-06, 2.633850954225636e-06, 2.3536692879133625e-06, 2.1543194179685088e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 5.229478801993537e-05, "first_to_last_ratio": 1.1054915648429926, "imbalance_abs_log_ratio": 0.10029009111257024, "last_quartile_mean": 4.730455634671671e-05, "mean": 5.2836898191799264e-05, "normalized": [ 0.9134570163319836, 1.034916113375725, 0.9380017327997141, 0.8761729806028853, 1.1651466691540098, 1.0805362503376124, 0.9982405315835244, 0.9114481612835588, 0.7967923600473241, 0.7497841466604215, 0.7143664402753938, 0.6965378517206229, 0.8463935773115121, 0.7846892424807549, 0.7343509538807638, 0.7009880274933418, 0.7821058780230875, 0.7327282264410435, 0.7471503818087507, 0.8667843236007904, 2.5845341323849897, 2.1052324444751704, 1.6964455897801056, 1.3808452078297222, 1.9978482082808935, 1.4674343240817393, 0.9715106963943513, 0.788467068735107, 0.6417902171752956, 0.4665439160451822, 0.42394890437146315, 0.4048084252331591 ], "peak_layer": 21, "peak_normalized": 2.5845341323849897, "population_cv": 0.484080554395872, "rest_mean": 4.3512348889230954e-05, "spike_contrast": 2.3714983690803897, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00010318946442566812, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 4.826423537451774e-05, 5.468175731948577e-05, 4.956110205966979e-05, 4.629426257451996e-05, 6.156273593660444e-05, 5.7092183851636946e-05, 5.274393333820626e-05, 4.815809370484203e-05, 4.2100036807823926e-05, 3.961626862292178e-05, 3.774490687646903e-05, 3.6802899558097124e-05, 4.4720811274601147e-05, 4.146054561715573e-05, 3.8800826587248594e-05, 3.703803304233588e-05, 4.132404865231365e-05, 3.8715086702723056e-05, 3.947710865759291e-05, 4.579819506034255e-05, 0.00013655876682605594, 0.00011123395233880728, 8.96349229151383e-05, 7.295957766473293e-05, 0.00010556010238360614, 7.753467798465863e-05, 5.1331611757632345e-05, 4.166015423834324e-05, 3.3910204365383834e-05, 2.4650733394082636e-05, 2.240014509879984e-05, 2.138882155122701e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 4.019236212116084e-05, "first_to_last_ratio": 1.4153153693492624, "imbalance_abs_log_ratio": 0.34735238212726816, "last_quartile_mean": 2.839816693267494e-05, "mean": 3.6624555320941e-05, "normalized": [ 1.0232717143864025, 1.1362712410352243, 1.0454296334488065, 0.9835124193343098, 1.2909201460459492, 1.1940430459228804, 1.0995455356552783, 1.0063318874214844, 0.8995783371506785, 0.8426433023808404, 0.7975342014164198, 0.7750456071733158, 0.9449206689124205, 0.8634790173788714, 0.8033806657557527, 0.7616815222194636, 0.8781075946878787, 0.8180653654670115, 0.821103325733316, 0.9180488653876261, 2.320608687309257, 1.875926955333867, 1.4964942208646947, 1.2009681530339829, 1.728587471725061, 1.233434416412248, 0.8915343579352191, 0.7609921603227805, 0.4924262018851847, 0.392089329493756, 0.36068109756253997, 0.34334285120747854 ], "peak_layer": 21, "peak_normalized": 2.320608687309257, "population_cv": 0.4109218127802722, "rest_mean": 3.171064485353849e-05, "spike_contrast": 1.9917498409959573, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.315967184491455e-05, "top_five_layers": [ 21, 22, 25, 23, 5 ], "top_five_spike_overlap": 4 }, "values": [ 3.7476871511898935e-05, 4.161542892688885e-05, 3.828839544439688e-05, 3.602070501074195e-05, 4.72793763037771e-05, 4.373129559098743e-05, 4.027036629850045e-05, 3.685645788209513e-05, 3.294665657449514e-05, 3.08614362438675e-05, 2.920933548011817e-05, 2.8385700716171414e-05, 3.460729931248352e-05, 3.162453504046425e-05, 2.9423459636745974e-05, 2.7896247047465295e-05, 3.216030017938465e-05, 2.9961280233692378e-05, 3.007254417752847e-05, 3.3623131457716227e-05, 8.499126124661416e-05, 6.870499055366963e-05, 5.480843537952751e-05, 4.398492455948144e-05, 6.330874748528004e-05, 4.517398701864295e-05, 3.265204941271804e-05, 2.7870999474544078e-05, 1.8034890672424808e-05, 1.436009733879473e-05, 1.3209784810896963e-05, 1.2574779248097911e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.0001358363469989854, "first_to_last_ratio": 0.8633942496929143, "imbalance_abs_log_ratio": 0.14688385595600487, "last_quartile_mean": 0.00015732829706394114, "mean": 0.00015515230734308716, "normalized": [ 0.7907368753073346, 0.9264540939215085, 0.8265322723723958, 0.7582197138401158, 1.0375539461851868, 0.9618311824030241, 0.8978516348790245, 0.8048461612150729, 0.6996799130387102, 0.653990298687687, 0.6297130190914373, 0.6204760902703826, 0.7480574883080816, 0.7062468888445648, 0.65916419436094, 0.6381152065686453, 0.6956919128206926, 0.6592207973273458, 0.6808477731568826, 0.800328991259754, 2.851733418364401, 2.3508709793179583, 1.916872822572222, 1.5727654287839425, 2.222955778533883, 1.7091773585367538, 1.110141083831678, 0.8952886037887667, 0.7625030624279955, 0.5112516860866544, 0.4612208540006556, 0.43966046989630203 ], "peak_layer": 21, "peak_normalized": 2.851733418364401, "population_cv": 0.5801733072620557, "rest_mean": 0.00012116131903086271, "spike_contrast": 2.795476699480493, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00033870364422909914, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 0.00012268465070519596, 0.00014374149031937122, 0.00012823838915210217, 0.00011763953807530925, 0.00016097888874355704, 0.00014923032722435892, 0.0001393037528032437, 0.00012487373896874487, 0.00010855695290956646, 0.00010146810382138938, 9.770142787601799e-05, 9.62682970566675e-05, 0.00011606284533627331, 0.00010957583435811102, 0.000102270845673047, 9.9005046649836e-05, 0.0001079382054740563, 0.00010227962775388733, 0.00010563510295469314, 0.00012417288962751627, 0.0004424530197866261, 0.0003647430567070842, 0.00029740724130533636, 0.00024401818518526852, 0.0003448967181611806, 0.0002651828108355403, 0.0001722409506328404, 0.00013890609261579812, 0.00011830410949187353, 7.932187872938812e-05, 7.155947969295084e-05, 6.821433635195717e-05 ] } } }, "pre_attention_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "ee695e78937de01b7128d3866b8fc5dd328ebdf2c159be37918d4f02952d5248", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.0626930519938469, 0.09030644595623016, 0.20229491591453552, 0.20300470292568207, 0.19049881398677826, 0.12695962190628052, 0.12132706493139267, 0.11585728824138641, 0.13427142798900604, 0.10485932976007462, 0.10419609397649765, 0.10852954536676407, 0.1286265105009079, 0.09096142649650574, 0.10080574452877045, 0.10566592961549759, 0.1287592649459839, 0.09650235623121262, 0.1065765842795372, 0.11446042358875275, 0.12395420670509338, 0.07221997529268265, 0.07677769660949707, 0.08416415005922318, 0.09094838052988052, 0.059439923614263535, 0.062182098627090454, 0.06896103918552399, 0.0776243805885315, 0.06285123527050018, 0.07231374084949493, 0.08016443997621536 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.0806050511291687e-05, "first_to_last_ratio": 1.2533588492728658, "imbalance_abs_log_ratio": 0.22582702698885568, "last_quartile_mean": 8.621673288189413e-06, "mean": 1.4104223332367383e-05, "normalized": [ 0.7154995788959891, 0.9131483356031458, 0.6475409461740426, 0.56464675625221, 0.3716231363352735, 0.8020314965680179, 1.0199078715425998, 1.0948584887353912, 0.7182859907835634, 0.6231487025717003, 0.5582992879188372, 0.5121353601249309, 0.46716586678969324, 0.791984395983001, 0.7497629466412109, 0.7042937679083474, 0.7664044887268359, 1.0736516831195402, 1.335784102050171, 1.7335838470000988, 2.447885057922223, 3.8299938460184983, 2.578083379174008, 2.090015908857374, 1.13669200320144, 1.1532349494717378, 0.7365750308978298, 0.534367620255654, 0.4377561343772761, 0.2674822091699633, 0.2989633725991937, 0.3251934383302024 ], "peak_layer": 22, "peak_normalized": 3.8299938460184983, "population_cv": 0.7623958121539145, "rest_mean": 1.0404387694507022e-05, "spike_contrast": 3.2758617592467805, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.408333577681333e-05, "top_five_layers": [ 22, 23, 21, 24, 20 ], "top_five_spike_overlap": 4 }, "values": [ 1.0091565854963847e-05, 1.287924806092633e-05, 9.133062121691182e-06, 7.963903954077978e-06, 5.241455710347509e-06, 1.1312031347188167e-05, 1.4385008398676291e-05, 1.5442128642462194e-05, 1.0130866030522157e-05, 8.789028470346238e-06, 7.87437784310896e-06, 7.223271495604422e-06, 6.589011718460824e-06, 1.1170324796694331e-05, 1.0574824045761488e-05, 9.933516594173852e-06, 1.0809540071932133e-05, 1.5143023119890131e-05, 1.8840197299141437e-05, 2.4450853743474e-05, 3.45255175489001e-05, 5.401908856583759e-05, 3.636186374933459e-05, 2.9478051146725193e-05, 1.603215787326917e-05, 1.6265483282040805e-05, 1.0388818736828398e-05, 7.536840257671429e-06, 6.174210284370929e-06, 3.7726288155681686e-06, 4.216646175336791e-06, 4.586600880429614e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 4.285675140636158e-05, "first_to_last_ratio": 1.258279937352894, "imbalance_abs_log_ratio": 0.22974565924079532, "last_quartile_mean": 3.405979077797383e-05, "mean": 5.635206036913587e-05, "normalized": [ 0.6883906893057068, 0.9159282922733917, 0.6137835398836017, 0.5705621229299551, 0.3827085434390855, 0.8202293842131958, 0.9960834309025619, 1.0964573097373815, 0.693528347583496, 0.6359598736651527, 0.5701664466767249, 0.5354174660558091, 0.47853931130466937, 0.8250703982776398, 0.7498260609587517, 0.7125475993269138, 0.7429088601911122, 1.1258353515875978, 1.3125701401715528, 1.7383828631559202, 2.456276578685518, 3.8356506083541557, 2.5933968317514378, 2.074493991275162, 1.1165830218012616, 1.1728079137025993, 0.7300268189246621, 0.47749360010384967, 0.4328798920888081, 0.27470619558406234, 0.30069584392358456, 0.3300926721646785 ], "peak_layer": 22, "peak_normalized": 3.8356506083541557, "population_cv": 0.76423967305293, "rest_mean": 4.158280932676496e-05, "spike_contrast": 3.2731318109941543, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001361060159979388, "top_five_layers": [ 22, 23, 21, 24, 20 ], "top_five_spike_overlap": 4 }, "values": [ 3.879223368130624e-05, 5.161444641998969e-05, 3.4587967093102634e-05, 3.2152351195691153e-05, 2.1566414943663403e-05, 4.622161577572115e-05, 5.613135363091715e-05, 6.178762851050124e-05, 3.9081751310732216e-05, 3.583764919312671e-05, 3.213005402358249e-05, 3.0171877369866706e-05, 2.696667615964543e-05, 4.6494416892528534e-05, 4.225424345349893e-05, 4.0153525333153084e-05, 4.1864444938255474e-05, 6.344314169837162e-05, 7.396603177767247e-05, 9.796145604923368e-05, 0.00013841624604538083, 0.00021614681463688612, 0.00014614325482398272, 0.00011690201063174754, 6.292175385169685e-05, 6.609014235436916e-05, 4.113851537113078e-05, 2.690774817892816e-05, 2.4393673811573535e-05, 1.5480260117328726e-05, 1.6944830349530093e-05, 1.860140218923334e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.0380057948641479, "first_to_last_ratio": 1.2582799192025502, "imbalance_abs_log_ratio": 0.22974564481606896, "last_quartile_mean": 0.030204562819562852, "mean": 0.04997357030515559, "normalized": [ 0.6883906500456527, 0.9159282748608111, 0.6137835301363085, 0.5705621427679602, 0.382708548738294, 0.8202293704748848, 0.9960834593053718, 1.0964572429078634, 0.6935283804718058, 0.6359598728748151, 0.5701664195200788, 0.5354174680070493, 0.47853928632830406, 0.8250704109647906, 0.7498260413268011, 0.7125476217442394, 0.7429089167352664, 1.1258353611227188, 1.3125701433664139, 1.7383829689479697, 2.4562764513449955, 3.835650732024849, 2.5933968490011075, 2.0744939032067173, 1.1165830332792048, 1.1728078634639203, 0.730026833520962, 0.4774936033934531, 0.4328798978632315, 0.2747061996952579, 0.30069584674815397, 0.33009267581074747 ], "peak_layer": 22, "peak_normalized": 3.835650732024849, "population_cv": 0.7642396792695201, "rest_mean": 0.03687605099683559, "spike_contrast": 3.273131783564383, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.12070017457008361, "top_five_layers": [ 22, 23, 21, 24, 20 ], "top_five_spike_overlap": 4 }, "values": [ 0.034401338547468185, 0.04577220603823662, 0.0306729543954134, 0.028513027355074883, 0.019125312566757202, 0.04098979011178017, 0.04977784678339958, 0.05479388311505318, 0.03465808928012848, 0.031781185418367386, 0.028493251651525497, 0.02675672248005867, 0.023914316669106483, 0.04123171418905258, 0.03747148439288139, 0.035608548671007156, 0.037125810980796814, 0.05626201257109642, 0.06559381633996964, 0.08687320351600647, 0.12274890393018723, 0.1916811615228653, 0.12960129976272583, 0.10366986691951752, 0.05579964071512222, 0.05860939621925354, 0.03648204728960991, 0.02386206015944481, 0.02163255400955677, 0.013728049583733082, 0.015026845037937164, 0.01649590954184532 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 3.4865606579614905e-06, "first_to_last_ratio": 0.5744714159436656, "imbalance_abs_log_ratio": 0.55430493759838, "last_quartile_mean": 6.069162992616839e-06, "mean": 4.761957452359411e-06, "normalized": [ 2.7549279941875353, 1.0962458945618736, 0.18443395211736102, 0.2418746565407424, 0.2301953185658598, 0.36104626586002875, 0.43696711947466477, 0.551665892437493, 0.4246542140373175, 0.42793597794286137, 0.39655368981734584, 0.4616801180942128, 0.34538018946761495, 0.5812142298006445, 0.6362219138883849, 0.6824319916360405, 0.38075941019660153, 0.7419672312235042, 0.6000574782283384, 1.0443385321943304, 1.6085285192233254, 2.8287190030338616, 2.6009399214014985, 2.1851794780624996, 1.6089985499792605, 2.230767304610303, 1.8064987507127341, 1.3993798256864782, 1.1611758400511873, 0.8526036091523058, 0.5554000203598723, 0.5812571074539186 ], "peak_layer": 22, "peak_normalized": 2.8287190030338616, "population_cv": 0.7787727801746858, "rest_mean": 3.7333101848465353e-06, "spike_contrast": 2.7634062497148295, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.031665269692894e-05, "top_five_layers": [ 22, 1, 23, 26, 24 ], "top_five_spike_overlap": 3 }, "values": [ 1.3118849892634898e-05, 5.220276307227323e-06, 8.782666327533661e-07, 1.1517968232510611e-06, 1.0961803127429448e-06, 1.719286956358701e-06, 2.080818831018405e-06, 2.6270095077052247e-06, 2.0221852992108325e-06, 2.0378129192977212e-06, 1.8883717984863324e-06, 2.1985010789649095e-06, 1.6446857671326143e-06, 2.7677174330165144e-06, 3.029661684195162e-06, 3.2497121082997182e-06, 1.8131601109416806e-06, 3.5332163861312438e-06, 2.8574481802934315e-06, 4.97309565616888e-06, 7.659744369448163e-06, 1.3470239537127782e-05, 1.2385565241856966e-05, 1.0405731700302567e-05, 7.661982635909226e-06, 1.0622818990668748e-05, 8.60247018863447e-06, 6.663787189609138e-06, 5.529469945031451e-06, 4.060062110511353e-06, 2.6447912659932626e-06, 2.7679216145770624e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 2.8424734864529455e-05, "first_to_last_ratio": 1.2835143350237113, "imbalance_abs_log_ratio": 0.2496018899645548, "last_quartile_mean": 2.2146020569380198e-05, "mean": 3.753192171984665e-05, "normalized": [ 0.537164752536801, 0.9214302641867693, 0.6444378068926937, 0.5953396754966896, 0.43371879550655473, 0.8753402179123481, 1.0472792752640416, 1.0040751335226212, 0.7433375894878967, 0.6792093070911899, 0.5962090914342423, 0.5738431616755357, 0.5338905453954232, 0.8983087146118869, 0.787309370397325, 0.7092189145683365, 0.7558827879624759, 1.1421229143369944, 1.2988621442736157, 1.640874351535352, 2.2280225964329383, 3.790745377612447, 2.754110595419175, 2.088800505558534, 1.1076448220255568, 1.2425992675575712, 0.7134643628819397, 0.4368730514552549, 0.44397595527151484, 0.2763345431766624, 0.23172675438783716, 0.26784735413177513 ], "peak_layer": 22, "peak_normalized": 3.790745377612447, "population_cv": 0.755959287150149, "rest_mean": 2.7844065462650854e-05, "spike_contrast": 3.2267682184995934, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 8.984634550870396e-05, "top_five_layers": [ 22, 23, 21, 24, 20 ], "top_five_spike_overlap": 4 }, "values": [ 2.016082544287201e-05, 3.4583048545755446e-05, 2.4186989321606234e-05, 2.2344242097460665e-05, 1.627829988137819e-05, 3.285320053691976e-05, 3.930640377802774e-05, 3.7684869312215596e-05, 2.7898888220079243e-05, 2.5492030545137823e-05, 2.2376872948370874e-05, 2.1537436623475514e-05, 2.003793815674726e-05, 3.37152523570694e-05, 2.9549233659054153e-05, 2.6618348783813417e-05, 2.8369733627187088e-05, 4.286606781533919e-05, 4.874879232374951e-05, 6.158516771392897e-05, 8.362196967937052e-05, 0.00014227395877242088, 0.00010336706327507272, 7.839669706299901e-05, 4.157203875365667e-05, 4.663713843910955e-05, 2.6777688617585227e-05, 1.6396685168729164e-05, 1.6663270798744634e-05, 1.0371366442996077e-05, 8.697150406078435e-06, 1.0052825928141829e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 1.880904278550588e-05, "first_to_last_ratio": 1.3122304557342896, "imbalance_abs_log_ratio": 0.2717283273333507, "last_quartile_mean": 1.433364292324768e-05, "mean": 2.4530393162081054e-05, "normalized": [ 0.41950892312696814, 0.9561362698230794, 0.6784941963877744, 0.6230469045100516, 0.49313699335602096, 0.9407843342993947, 1.1235667038119015, 0.8994443286265796, 0.7834963277471938, 0.7479964228219654, 0.6342003268118622, 0.606856897696476, 0.5932137318422519, 0.9764984657293563, 0.8306142202397758, 0.7225105873964619, 0.7702020492951054, 1.1549696102556435, 1.290309151123356, 1.5617267915251511, 1.9594901672196914, 3.6687984802700195, 2.862607119704501, 2.027816780290767, 1.0632060688384286, 1.3284601553899558, 0.7090835018588847, 0.4037051765637059, 0.4554505543549642, 0.2874645514737616, 0.20110077486030092, 0.2261034327486491 ], "peak_layer": 22, "peak_normalized": 3.6687984802700195, "population_cv": 0.7327080825339415, "rest_mean": 1.8550502368776102e-05, "spike_contrast": 3.0630870429454955, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.68218034459278e-05, "top_five_layers": [ 22, 23, 24, 21, 20 ], "top_five_spike_overlap": 4 }, "values": [ 1.0290718819305766e-05, 2.3454398615285754e-05, 1.664372939558234e-05, 1.5283585526049137e-05, 1.2096844329789747e-05, 2.307780960109085e-05, 2.7561532988329418e-05, 2.2063723008614033e-05, 1.921947296068538e-05, 1.834864633565303e-05, 1.5557183360215276e-05, 1.4886438293615356e-05, 1.4551766071235761e-05, 2.3953891286510043e-05, 2.0375293388497084e-05, 1.7723468772601336e-05, 1.8893359083449468e-05, 2.8331858629826456e-05, 3.165179077768698e-05, 3.8309772207867354e-05, 4.806706419913098e-05, 8.999706915346906e-05, 7.022087811492383e-05, 4.9743142881197855e-05, 2.6080862880917266e-05, 3.2587649911874905e-05, 1.7394097085343674e-05, 9.903046702675056e-06, 1.117238116421504e-05, 7.051618467812659e-06, 4.933081072522327e-06, 5.546406100620516e-06 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 8.418645074925735e-05, "first_to_last_ratio": 1.2777405535512787, "imbalance_abs_log_ratio": 0.24509332560529284, "last_quartile_mean": 6.588696783182968e-05, "mean": 0.00011224537939824586, "normalized": [ 0.6520524645870711, 0.8827495514367005, 0.6384170504233534, 0.5865450832999949, 0.3805023216068713, 0.8557844231415733, 0.9418795607393456, 1.0622417532309283, 0.7015575900480739, 0.6089772168412393, 0.566684239661543, 0.554400232256426, 0.4842878066849515, 0.8379536066260636, 0.7607829523506184, 0.687027244873744, 0.722054855000606, 1.1324008181521155, 1.3067624970122615, 1.6601357881756402, 2.4369003829913747, 3.9705411796812258, 2.6987629269515483, 2.1746747068045282, 1.1456404298566998, 1.1780775628013878, 0.7097854325206946, 0.44575553626403225, 0.43467912825498933, 0.24890787307768844, 0.23821837072340374, 0.2948594139233055 ], "peak_layer": 22, "peak_normalized": 3.9705411796812258, "population_cv": 0.7927154396003535, "rest_mean": 8.137158261821174e-05, "spike_contrast": 3.4282715542022077, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0002789638820104301, "top_five_layers": [ 22, 23, 21, 24, 20 ], "top_five_spike_overlap": 4 }, "values": [ 7.318987627513707e-05, 9.90845583146438e-05, 7.165936403907835e-05, 6.583697540918365e-05, 4.2709627450676635e-05, 9.605784725863487e-05, 0.00010572162864264101, 0.00011923172860406339, 7.874659786466509e-05, 6.835487874923274e-05, 6.360768747981638e-05, 6.222886440809816e-05, 5.4359068599296734e-05, 9.405642049387097e-05, 8.539437112631276e-05, 7.711563375778496e-05, 8.104732114588842e-05, 0.00012710675946436822, 0.00014667805226054043, 0.00018634257139638066, 0.0002735308080445975, 0.0004456749011296779, 0.00030292366864159703, 0.00024409718753304332, 0.00012859284470323473, 0.00013223376299720258, 7.96701351646334e-05, 5.003399928682484e-05, 4.879072366748005e-05, 2.7938758648815565e-05, 2.6738911401480436e-05, 3.309660678496584e-05 ] } } }, "pre_mlp_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "e53fa425d89dfeb0e9395eca9bcd73dc0a55314abf45c5b10b43e2cbfa724345", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.059514161199331284, 0.08157797157764435, 0.14292451739311218, 0.10997853428125381, 0.0991024449467659, 0.1065012738108635, 0.12155740708112717, 0.1095810979604721, 0.06964563578367233, 0.08311934769153595, 0.09363401681184769, 0.0941167026758194, 0.08564268052577972, 0.09493856877088547, 0.09063401818275452, 0.09473779052495956, 0.10505300015211105, 0.09355910867452621, 0.09618847072124481, 0.11416846513748169, 0.047205954790115356, 0.053352247923612595, 0.06512638926506042, 0.07829347252845764, 0.05368548259139061, 0.05310077592730522, 0.061235878616571426, 0.07320034503936768, 0.05304349213838577, 0.0639304369688034, 0.07404901832342148, 0.08157160878181458 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 5.481314360622491e-06, "first_to_last_ratio": 0.1735780896016552, "imbalance_abs_log_ratio": 1.7511276966933207, "last_quartile_mean": 3.15783770474809e-05, "mean": 1.7297920990699822e-05, "normalized": [ 0.651413334162405, 0.24266681365140175, 0.15763378385697482, 0.29446011303717745, 0.21815591675347784, 0.17950714458831163, 0.24274221088573086, 0.5484371690662797, 0.4969404899716252, 0.37616429447096716, 0.3386371087447132, 0.3461648443328364, 0.3877018852735207, 0.41194601319119534, 0.4828601879063288, 0.5056894609584669, 0.34896200811687345, 0.44511945555020893, 0.42348667982234595, 0.5743243415266098, 2.3513127019068594, 2.7776896353342884, 2.446489038885117, 2.147022581035446, 2.7951027159856516, 2.826526853125176, 2.395433649182181, 1.5839323748456084, 1.7976864785515911, 1.423159813061818, 0.9260453416059562, 0.8565855606128555 ], "peak_layer": 26, "peak_normalized": 2.826526853125176, "population_cv": 0.901936180177488, "rest_mean": 1.2481656574015721e-05, "spike_contrast": 3.4695513839843786, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.3305748840793966e-05, "top_five_layers": [ 26, 25, 22, 23, 27 ], "top_five_spike_overlap": 3 }, "values": [ 1.1268096386629622e-05, 4.197631369606825e-06, 2.7267367386230035e-06, 5.093547770229634e-06, 3.7736438116553472e-06, 3.1051004043547437e-06, 4.198935585009167e-06, 9.486822818871588e-06, 8.59603733260883e-06, 6.506860245281132e-06, 5.857717951585073e-06, 5.987932127027307e-06, 6.706436579406727e-06, 7.125809588615084e-06, 8.352477379958145e-06, 8.747376341489144e-06, 6.0363172451616265e-06, 7.699641173530836e-06, 7.325439128180733e-06, 9.934617082762998e-06, 4.0672821342013776e-05, 4.804825584869832e-05, 4.23191740992479e-05, 3.7139026971999556e-05, 4.8349465942010283e-05, 4.88930381834507e-05, 4.1436022002017125e-05, 2.739873707469087e-05, 3.1096238672034815e-05, 2.4617706003482454e-05, 1.6018659152905457e-05, 1.4817149349255487e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 2.1982303337608755e-05, "first_to_last_ratio": 0.17392514236767914, "imbalance_abs_log_ratio": 1.7491302886148878, "last_quartile_mean": 0.00012638945145226899, "mean": 6.915283486819135e-05, "normalized": [ 0.654139380373682, 0.24479524019776733, 0.15775005147108534, 0.2951814695792291, 0.21539928801740166, 0.178283521955047, 0.24000401072036412, 0.5574870596319528, 0.5001161446745576, 0.3805530988338838, 0.3373053276404746, 0.3467657101852835, 0.39042950389550374, 0.41777204195252265, 0.4857194929357486, 0.5058368225353395, 0.3486899984528497, 0.447450734777375, 0.42372608803340744, 0.5755127029305313, 2.3468381503098783, 2.757330324296252, 2.4388430976152033, 2.132607807705165, 2.804614639709741, 2.8218588466824834, 2.4027074280998666, 1.5901305437717372, 1.8023454813917477, 1.41737472109916, 0.9299993975214507, 0.852431873003307 ], "peak_layer": 26, "peak_normalized": 2.8218588466824834, "population_cv": 0.8998679093738083, "rest_mean": 4.9994339018734604e-05, "spike_contrast": 3.452565147237477, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00017260871245525778, "top_five_layers": [ 26, 25, 22, 23, 27 ], "top_five_spike_overlap": 3 }, "values": [ 4.523559255176224e-05, 1.692828482191544e-05, 1.090886325982865e-05, 2.0412635421962477e-05, 1.4895471394993365e-05, 1.2328810953476932e-05, 1.6596957721048966e-05, 3.855181057588197e-05, 3.4584449167596176e-05, 2.6316325602238066e-05, 2.3325619622482918e-05, 2.397983189439401e-05, 2.699930701055564e-05, 2.8890121029689908e-05, 3.358887988724746e-05, 3.498005025903694e-05, 2.411290188319981e-05, 3.09424867737107e-05, 2.9301860195118934e-05, 3.979833491030149e-05, 0.00016229051107075065, 0.0001906772085931152, 0.00016865291399881244, 0.00014747587556485087, 0.0001939470530487597, 0.00019513953884597868, 0.00016615403001196682, 0.00010996203491231427, 0.00012463729945011437, 9.801548003451899e-05, 6.431209476431832e-05, 5.894808055018075e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.01949412631802261, "first_to_last_ratio": 0.1739251392210055, "imbalance_abs_log_ratio": 1.7491303067070028, "last_quartile_mean": 0.11208343086764216, "mean": 0.061325426562689245, "normalized": [ 0.6541393565482417, 0.24479523245432414, 0.15775006392926896, 0.29518147587301974, 0.21539929722993656, 0.1782835146896786, 0.24000400788636042, 0.5574870151594931, 0.5001161515946682, 0.3805530849387255, 0.3373053319424034, 0.34676568109599726, 0.3904294852261518, 0.41777204500136844, 0.48571947024485224, 0.5058368091251695, 0.3486899707429497, 0.44745075406543916, 0.42372608966228065, 0.5755127388200528, 2.3468382007403554, 2.7573303228398136, 2.438843271010461, 2.132607767856224, 2.804614482565551, 2.821858887831695, 2.4027075078806455, 1.59013042455487, 1.8023455395216372, 1.4173747435803081, 0.929999421410858, 0.852431853977199 ], "peak_layer": 26, "peak_normalized": 2.821858887831695, "population_cv": 0.89986791701269, "rest_mean": 0.04433548050346198, "spike_contrast": 3.4525651587460247, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.15307113528251648, "top_five_layers": [ 26, 25, 22, 23, 27 ], "top_five_spike_overlap": 3 }, "values": [ 0.04011537507176399, 0.015012172050774097, 0.009674089960753918, 0.0181021299213171, 0.013209453783929348, 0.010933312587440014, 0.014718348160386086, 0.034188129007816315, 0.030669836327433586, 0.023337580263614655, 0.02068539336323738, 0.021265553310513496, 0.023943254724144936, 0.025620048865675926, 0.029786953702569008, 0.03102065809071064, 0.021383561193943024, 0.027440108358860016, 0.02598518319427967, 0.035293564200401306, 0.14392085373401642, 0.16909445822238922, 0.14956310391426086, 0.1307830810546875, 0.1719941794872284, 0.17305169999599457, 0.14734706282615662, 0.09751542657613754, 0.11052960902452469, 0.08692111074924469, 0.05703261122107506, 0.05227574706077576 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 1.2048782807028147e-06, "first_to_last_ratio": 0.1418162776729321, "imbalance_abs_log_ratio": 1.9532228782922858, "last_quartile_mean": 8.49605066832737e-06, "mean": 4.476531985986298e-06, "normalized": [ 0.6113949368766554, 0.19753118972333122, 0.1513866631088621, 0.30591165986873536, 0.1999787979149841, 0.1575976812707844, 0.1911475028946622, 0.3382867682237939, 0.4387344984941179, 0.3377085478671989, 0.3190957595376408, 0.3141337257896916, 0.36283279123581025, 0.4103370169359222, 0.5091661974975977, 0.4887724508516774, 0.34916890682618135, 0.44047134363587515, 0.4218525618064859, 0.40691729354707407, 2.3883284149327455, 2.8234654421341236, 2.4983146224110997, 2.1541926487817475, 2.853232616177045, 2.887074354257743, 2.655541798728337, 1.673223328371301, 1.8223500357048348, 1.4200255915538205, 0.9814235087720745, 0.8904013442680457 ], "peak_layer": 26, "peak_normalized": 2.887074354257743, "population_cv": 0.946855900492777, "rest_mean": 3.1969843319159366e-06, "spike_contrast": 3.5615092649336275, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.1386089317966252e-05, "top_five_layers": [ 26, 25, 22, 27, 23 ], "top_five_spike_overlap": 3 }, "values": [ 2.736928990998422e-06, 8.842546890264202e-07, 6.776872396585532e-07, 1.369423330288555e-06, 8.952114853855164e-07, 7.054910611259402e-07, 8.556779107493639e-07, 1.5143515383897466e-06, 1.9640090158645762e-06, 1.5117631164685008e-06, 1.4284423741628416e-06, 1.4062296713746036e-06, 1.6242325955317938e-06, 1.8368867813478573e-06, 2.279298769281013e-06, 2.18800551010645e-06, 1.5630657799192704e-06, 1.9717840586963575e-06, 1.888436486296996e-06, 1.8215782802144531e-06, 1.0691428542486392e-05, 1.263933336304035e-05, 1.118378531828057e-05, 9.643312296248041e-06, 1.277258706977591e-05, 1.2924080692755524e-05, 1.188761780213099e-05, 7.490237749152584e-06, 8.157808224495966e-06, 6.356789981509792e-06, 4.393373728817096e-06, 3.985910097981105e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 1.4753988523352746e-05, "first_to_last_ratio": 0.19556335753154602, "imbalance_abs_log_ratio": 1.6318708726138689, "last_quartile_mean": 7.544352229160722e-05, "mean": 4.350447703416194e-05, "normalized": [ 0.7190627436507564, 0.25827238096103255, 0.12927292468587806, 0.2733040077282239, 0.24950355919389386, 0.20349787709964412, 0.2642013207734653, 0.6159831526781414, 0.575493772257737, 0.433329167041521, 0.3918205465862251, 0.4222431703507383, 0.467915969828678, 0.48199320626742714, 0.5504920989384697, 0.5826962290165841, 0.39704652979667177, 0.5207246620856664, 0.4807697586462694, 0.6055842032964343, 2.2594875348702366, 2.6850996286558253, 2.4408031329482207, 2.1181594950483724, 2.9032802988018953, 2.692944477993582, 2.258948667676994, 1.521278950226039, 1.597602915374006, 1.2862686314388623, 0.8413345028087829, 0.7715844832737266 ], "peak_layer": 25, "peak_normalized": 2.9032802988018953, "population_cv": 0.8581712312392323, "rest_mean": 3.157002260599661e-05, "spike_contrast": 3.419399862125849, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00010795053094625473, "top_five_layers": [ 25, 26, 22, 23, 21 ], "top_five_spike_overlap": 4 }, "values": [ 3.1282448617275804e-05, 1.1236004866077565e-05, 5.623950983135728e-06, 1.1889947927556932e-05, 1.0854521860892419e-05, 8.853068720782176e-06, 1.1493940291984472e-05, 2.679802491911687e-05, 2.503655559848994e-05, 1.8851758795790374e-05, 1.7045947970473208e-05, 1.836946830735542e-05, 2.035643956332933e-05, 2.096886237268336e-05, 2.3948870875756256e-05, 2.534989471314475e-05, 1.7273301637033e-05, 2.265385410282761e-05, 2.0915636923746206e-05, 2.634562406456098e-05, 9.829782356973737e-05, 0.0001168138551292941, 0.00010618586384225637, 9.214942110702395e-05, 0.00012630569108296186, 0.00011715514119714499, 9.827438043430448e-05, 6.61824451526627e-05, 6.95028793415986e-05, 5.5958444136194885e-05, 3.6601817555492744e-05, 3.3567379432497546e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 9.786470513972745e-06, "first_to_last_ratio": 0.2372991508987558, "imbalance_abs_log_ratio": 1.4384336937231554, "last_quartile_mean": 4.124106840208697e-05, "mean": 2.58907955981158e-05, "normalized": [ 0.7993845326889925, 0.284081056329827, 0.11930373636864965, 0.2730899626733759, 0.30047516937418656, 0.2435030665840603, 0.30303137164336014, 0.7010537881765735, 0.6874429103728265, 0.512586176580009, 0.47077722762255064, 0.5295739546742398, 0.5821260279634725, 0.5808726570179454, 0.6405879138757905, 0.6852868172704004, 0.4743233400163025, 0.6270860034675879, 0.5610985524094696, 0.6478725001906962, 2.182064578361594, 2.56163448002043, 2.4037943293716006, 2.0858672341928903, 2.915552150002437, 2.4670035685087592, 1.9760709557776555, 1.3816141183191526, 1.3971630841780178, 1.1549168310942117, 0.7492539804972644, 0.7015079243756702 ], "peak_layer": 25, "peak_normalized": 2.915552150002437, "population_cv": 0.7881934057799003, "rest_mean": 1.9035571919327292e-05, "spike_contrast": 3.3048128908436243, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.290900346357375e-05, "top_five_layers": [ 25, 22, 26, 23, 21 ], "top_five_spike_overlap": 4 }, "values": [ 2.0696701540146023e-05, 7.3550845627323724e-06, 3.088868652412202e-06, 7.070516403473448e-06, 7.77954119257629e-06, 6.304488124442287e-06, 7.845723303034902e-06, 1.8150840332964435e-05, 1.779844387783669e-05, 1.3271263924252708e-05, 1.2188796972623095e-05, 1.3711091014556587e-05, 1.5071706002345309e-05, 1.503925523138605e-05, 1.65853307407815e-05, 1.774262091203127e-05, 1.2280608643777668e-05, 1.6235755538218655e-05, 1.4527287930832244e-05, 1.6773934476077557e-05, 5.649538798024878e-05, 6.632275471929461e-05, 6.223614764166996e-05, 5.4004762205295265e-05, 7.548596477136016e-05, 6.387268513208255e-05, 5.116204920341261e-05, 3.577108873287216e-05, 3.617366382968612e-05, 2.9901715606683865e-05, 1.9398781660129316e-05, 1.8162598280468956e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 4.332495768721856e-05, "first_to_last_ratio": 0.1607181804137053, "imbalance_abs_log_ratio": 1.8281028800072834, "last_quartile_mean": 0.0002695709817999159, "mean": 0.00014441376873719491, "normalized": [ 0.63883835422854, 0.23387313256386755, 0.12555140579074833, 0.2576695433817421, 0.20357721308195972, 0.17445707499767746, 0.2281076874477648, 0.5379714049733911, 0.4907299523853792, 0.37059204326178496, 0.3302276799925244, 0.3361957895758658, 0.3761397594791501, 0.40651544739306683, 0.47574321103152417, 0.4947017224988308, 0.34538179272243086, 0.43720617841789555, 0.41816189037993035, 0.551641696392012, 2.229898214318018, 2.736944407115424, 2.4684204622029338, 2.198197410642052, 2.9330984200454395, 2.8686297068100726, 2.593403248682991, 1.7502459092693121, 1.7303217627046539, 1.3582989259381137, 0.8835932965195947, 0.8156652557553092 ], "peak_layer": 25, "peak_normalized": 2.9330984200454395, "population_cv": 0.9256864150458745, "rest_mean": 0.00010394283210054575, "spike_contrast": 3.4918889474168435, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0003629568265751004, "top_five_layers": [ 25, 26, 22, 27, 23 ], "top_five_spike_overlap": 3 }, "values": [ 9.225705434801057e-05, 3.37745004799217e-05, 1.8131351680494845e-05, 3.7211029848549515e-05, 2.939935257018078e-05, 2.5194003683282062e-05, 3.2941890822257847e-05, 7.769047806505114e-05, 7.086816185619682e-05, 5.351859363145195e-05, 4.768942380906083e-05, 4.855130100622773e-05, 5.431976023828611e-05, 5.870642780791968e-05, 6.870387005619705e-05, 7.144174014683813e-05, 4.987788634025492e-05, 6.313859194051474e-05, 6.0388334532035515e-05, 7.966465636854991e-05, 0.00032202800503000617, 0.00039525245665572584, 0.00035647390177473426, 0.00031744997249916196, 0.0004235797969158739, 0.00041426962707191706, 0.00037452313699759543, 0.00025275960797443986, 0.00024988228688016534, 0.000196157066966407, 0.00012760303798131645, 0.00011779329361161217 ] } } } }, "selector": { "expected_uniform_count": 4, "identities_unique": true, "identity_order_sha256": "97fd9721a263024f6a746638f32e0ad9a988cd4c111c00e9e178c1022e5c9d5e", "passed": true, "uniform_count": 4, "uniform_indices": [ 40, 42, 44, 46 ], "visit_count": 65, "visits": [ { "branch": "attention", "coefficient": "detached_learned", "group": 1, "index": 0, "kind": "depth", "layer": 1, "offset": 1, "sources": 1, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 1, "index": 1, "kind": "depth", "layer": 1, "offset": 1, "sources": 2, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 1, "index": 2, "kind": "depth", "layer": 2, "offset": 2, "sources": 2, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 1, "index": 3, "kind": "depth", "layer": 2, "offset": 2, "sources": 2, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 1, "index": 4, "kind": "depth", "layer": 3, "offset": 3, "sources": 2, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 1, "index": 5, "kind": "depth", "layer": 3, "offset": 3, "sources": 2, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 1, "index": 6, "kind": "depth", "layer": 4, "offset": 4, "sources": 2, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 1, "index": 7, "kind": "depth", "layer": 4, "offset": 4, "sources": 2, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 2, "index": 8, "kind": "depth", "layer": 5, "offset": 1, "sources": 2, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 2, "index": 9, "kind": "depth", "layer": 5, "offset": 1, "sources": 3, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 2, "index": 10, "kind": "depth", "layer": 6, "offset": 2, "sources": 3, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 2, "index": 11, "kind": "depth", "layer": 6, "offset": 2, "sources": 3, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 2, "index": 12, "kind": "depth", "layer": 7, "offset": 3, "sources": 3, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 2, "index": 13, "kind": "depth", "layer": 7, "offset": 3, "sources": 3, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 2, "index": 14, "kind": "depth", "layer": 8, "offset": 4, "sources": 3, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 2, "index": 15, "kind": "depth", "layer": 8, "offset": 4, "sources": 3, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 3, "index": 16, "kind": "depth", "layer": 9, "offset": 1, "sources": 3, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 3, "index": 17, "kind": "depth", "layer": 9, "offset": 1, "sources": 4, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 3, "index": 18, "kind": "depth", "layer": 10, "offset": 2, "sources": 4, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 3, "index": 19, "kind": "depth", "layer": 10, "offset": 2, "sources": 4, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 3, "index": 20, "kind": "depth", "layer": 11, "offset": 3, "sources": 4, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 3, "index": 21, "kind": "depth", "layer": 11, "offset": 3, "sources": 4, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 3, "index": 22, "kind": "depth", "layer": 12, "offset": 4, "sources": 4, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 3, "index": 23, "kind": "depth", "layer": 12, "offset": 4, "sources": 4, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 4, "index": 24, "kind": "depth", "layer": 13, "offset": 1, "sources": 4, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 4, "index": 25, "kind": "depth", "layer": 13, "offset": 1, "sources": 5, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 4, "index": 26, "kind": "depth", "layer": 14, "offset": 2, "sources": 5, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 4, "index": 27, "kind": "depth", "layer": 14, "offset": 2, "sources": 5, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 4, "index": 28, "kind": "depth", "layer": 15, "offset": 3, "sources": 5, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 4, "index": 29, "kind": "depth", "layer": 15, "offset": 3, "sources": 5, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 4, "index": 30, "kind": "depth", "layer": 16, "offset": 4, "sources": 5, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 4, "index": 31, "kind": "depth", "layer": 16, "offset": 4, "sources": 5, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 5, "index": 32, "kind": "depth", "layer": 17, "offset": 1, "sources": 5, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 5, "index": 33, "kind": "depth", "layer": 17, "offset": 1, "sources": 6, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 5, "index": 34, "kind": "depth", "layer": 18, "offset": 2, "sources": 6, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 5, "index": 35, "kind": "depth", "layer": 18, "offset": 2, "sources": 6, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 5, "index": 36, "kind": "depth", "layer": 19, "offset": 3, "sources": 6, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 5, "index": 37, "kind": "depth", "layer": 19, "offset": 3, "sources": 6, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 5, "index": 38, "kind": "depth", "layer": 20, "offset": 4, "sources": 6, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 5, "index": 39, "kind": "depth", "layer": 20, "offset": 4, "sources": 6, "use_uniform": false }, { "branch": "attention", "coefficient": "uniform", "group": 6, "index": 40, "kind": "depth", "layer": 21, "offset": 1, "sources": 6, "use_uniform": true }, { "branch": "mlp", "coefficient": "detached_learned", "group": 6, "index": 41, "kind": "depth", "layer": 21, "offset": 1, "sources": 7, "use_uniform": false }, { "branch": "attention", "coefficient": "uniform", "group": 6, "index": 42, "kind": "depth", "layer": 22, "offset": 2, "sources": 7, "use_uniform": true }, { "branch": "mlp", "coefficient": "detached_learned", "group": 6, "index": 43, "kind": "depth", "layer": 22, "offset": 2, "sources": 7, "use_uniform": false }, { "branch": "attention", "coefficient": "uniform", "group": 6, "index": 44, "kind": "depth", "layer": 23, "offset": 3, "sources": 7, "use_uniform": true }, { "branch": "mlp", "coefficient": "detached_learned", "group": 6, "index": 45, "kind": "depth", "layer": 23, "offset": 3, "sources": 7, "use_uniform": false }, { "branch": "attention", "coefficient": "uniform", "group": 6, "index": 46, "kind": "depth", "layer": 24, "offset": 4, "sources": 7, "use_uniform": true }, { "branch": "mlp", "coefficient": "detached_learned", "group": 6, "index": 47, "kind": "depth", "layer": 24, "offset": 4, "sources": 7, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 7, "index": 48, "kind": "depth", "layer": 25, "offset": 1, "sources": 7, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 7, "index": 49, "kind": "depth", "layer": 25, "offset": 1, "sources": 8, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 7, "index": 50, "kind": "depth", "layer": 26, "offset": 2, "sources": 8, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 7, "index": 51, "kind": "depth", "layer": 26, "offset": 2, "sources": 8, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 7, "index": 52, "kind": "depth", "layer": 27, "offset": 3, "sources": 8, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 7, "index": 53, "kind": "depth", "layer": 27, "offset": 3, "sources": 8, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 7, "index": 54, "kind": "depth", "layer": 28, "offset": 4, "sources": 8, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 7, "index": 55, "kind": "depth", "layer": 28, "offset": 4, "sources": 8, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 8, "index": 56, "kind": "depth", "layer": 29, "offset": 1, "sources": 8, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 8, "index": 57, "kind": "depth", "layer": 29, "offset": 1, "sources": 9, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 8, "index": 58, "kind": "depth", "layer": 30, "offset": 2, "sources": 9, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 8, "index": 59, "kind": "depth", "layer": 30, "offset": 2, "sources": 9, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 8, "index": 60, "kind": "depth", "layer": 31, "offset": 3, "sources": 9, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 8, "index": 61, "kind": "depth", "layer": 31, "offset": 3, "sources": 9, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 8, "index": 62, "kind": "depth", "layer": 32, "offset": 4, "sources": 9, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 8, "index": 63, "kind": "depth", "layer": 32, "offset": 4, "sources": 9, "use_uniform": false }, { "branch": "output", "coefficient": "detached_learned", "group": null, "index": 64, "kind": "output", "layer": null, "offset": null, "sources": 9, "use_uniform": false } ], "visits_sha256": "405951ed108e29cdea8fcb883903e04869c6b74f99f62584e8719da66082d8cf" } }, "uniform_group_6_mlp_only": { "bits_per_byte": 1.8740129107275643, "forward": { "activation_sha256": { "attention_branch_output": "5e6ecbcd78543fc42d4098a51169dca446d404ce384015643e84d2c78433c949", "mlp_branch_output": "c0b8018c349b1bd09e5d17858eb312a2323da0e92de9651e2bb08ae4adf115c7", "post_attention_state": "90a633f12d984935028fe5d7212be48e7da7b4a0806751939a551938d16dc540", "post_mlp_state": "89f2226456bfe96e245d2aad0f699d32c676f9f53099789cd870253d002402f5", "pre_attention_input": "ee695e78937de01b7128d3866b8fc5dd328ebdf2c159be37918d4f02952d5248", "pre_mlp_input": "e53fa425d89dfeb0e9395eca9bcd73dc0a55314abf45c5b10b43e2cbfa724345" }, "logits_sha256": "bdacba5675f0579cec629bbf01b36c8660ad576cfdf9fef1bea8259eaeac9ec2", "loss_nats": 1.2989667654037476, "mixer_summary_sha256": "eb3ac0f07de9b755d6e67a510ab68d09d9ad666bb7b5d020ba7d8c5055c399b2" }, "loss_nats": 1.2989667654037476, "loss_scale": 1.0, "mixers": [ { "branch": "attention", "entropy_mean": -0.0, "group": 1, "latest_source_mass": 1.0, "layer": 1, "max_source_mass": 1.0, "mixer_index": 0, "normalized_entropy": 1.0, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 1.0, "median": 1.0, "p05": 1.0, "p95": 1.0 } ], "sources": 1 }, { "branch": "mlp", "entropy_mean": 0.22886337339878082, "group": 1, "latest_source_mass": 0.06350618600845337, "layer": 1, "max_source_mass": 0.9364938139915466, "mixer_index": 1, "normalized_entropy": 0.3301800538435402, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.9364938139915466, "median": 0.9452826976776123, "p05": 0.8725647330284119, "p95": 0.9712384939193726 }, { "label": "current_group_partial", "mean": 0.06350618600845337, "median": 0.0547172985970974, "p05": 0.028761589899659157, "p95": 0.12743529677391052 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.603729784488678, "group": 1, "latest_source_mass": 0.4133155345916748, "layer": 2, "max_source_mass": 0.5866844654083252, "mixer_index": 2, "normalized_entropy": 0.8709979661187783, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.5866844654083252, "median": 0.5734395384788513, "p05": 0.29813364148139954, "p95": 0.8843642473220825 }, { "label": "current_group_partial", "mean": 0.4133155345916748, "median": 0.4265604615211487, "p05": 0.11563576757907867, "p95": 0.7018663287162781 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.3967982828617096, "group": 1, "latest_source_mass": 0.14110636711120605, "layer": 2, "max_source_mass": 0.858893632888794, "mixer_index": 3, "normalized_entropy": 0.5724589149178446, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.858893632888794, "median": 0.8705973029136658, "p05": 0.7599387168884277, "p95": 0.9270997643470764 }, { "label": "current_group_partial", "mean": 0.14110636711120605, "median": 0.12940272688865662, "p05": 0.07290027290582657, "p95": 0.2400612235069275 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6852496862411499, "group": 1, "latest_source_mass": 0.518975019454956, "layer": 3, "max_source_mass": 0.518975019454956, "mixer_index": 4, "normalized_entropy": 0.9886063241108252, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.48102498054504395, "median": 0.48102718591690063, "p05": 0.38786420226097107, "p95": 0.580143928527832 }, { "label": "current_group_partial", "mean": 0.518975019454956, "median": 0.5189728140830994, "p05": 0.41985607147216797, "p95": 0.6121357679367065 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.5890734195709229, "group": 1, "latest_source_mass": 0.3123423159122467, "layer": 3, "max_source_mass": 0.6876576542854309, "mixer_index": 5, "normalized_entropy": 0.8498533011344741, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.6876576542854309, "median": 0.6909762620925903, "p05": 0.48803940415382385, "p95": 0.8900076746940613 }, { "label": "current_group_partial", "mean": 0.3123423159122467, "median": 0.3090237081050873, "p05": 0.10999235510826111, "p95": 0.5119606256484985 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6873323321342468, "group": 1, "latest_source_mass": 0.4831485152244568, "layer": 4, "max_source_mass": 0.5168514251708984, "mixer_index": 6, "normalized_entropy": 0.9916109470127238, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.5168514251708984, "median": 0.5219584703445435, "p05": 0.4251725673675537, "p95": 0.5947866439819336 }, { "label": "current_group_partial", "mean": 0.4831485152244568, "median": 0.47804147005081177, "p05": 0.405213326215744, "p95": 0.5748274326324463 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.487301766872406, "group": 1, "latest_source_mass": 0.2060384750366211, "layer": 4, "max_source_mass": 0.7939615249633789, "mixer_index": 7, "normalized_entropy": 0.7030278424832499, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.7939615249633789, "median": 0.7953404188156128, "p05": 0.6590759754180908, "p95": 0.9329437017440796 }, { "label": "current_group_partial", "mean": 0.2060384750366211, "median": 0.20465964078903198, "p05": 0.06705628335475922, "p95": 0.3409239649772644 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.674697756767273, "group": 2, "latest_source_mass": 0.42952626943588257, "layer": 5, "max_source_mass": 0.5704737305641174, "mixer_index": 8, "normalized_entropy": 0.9733831077870527, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.5704737305641174, "median": 0.5696132779121399, "p05": 0.4695202708244324, "p95": 0.6750621199607849 }, { "label": "completed_group_1", "mean": 0.42952626943588257, "median": 0.4303867816925049, "p05": 0.3249378800392151, "p95": 0.5304797887802124 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.9691112041473389, "group": 2, "latest_source_mass": 0.2681158185005188, "layer": 5, "max_source_mass": 0.5506307482719421, "mixer_index": 9, "normalized_entropy": 0.8821230329784768, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.5506307482719421, "median": 0.543993353843689, "p05": 0.41262108087539673, "p95": 0.7161091566085815 }, { "label": "completed_group_1", "mean": 0.18125343322753906, "median": 0.1794438362121582, "p05": 0.11575750261545181, "p95": 0.25031501054763794 }, { "label": "current_group_partial", "mean": 0.2681158185005188, "median": 0.2664111256599426, "p05": 0.1564716249704361, "p95": 0.3817881643772125 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0745112895965576, "group": 2, "latest_source_mass": 0.412794291973114, "layer": 6, "max_source_mass": 0.412794291973114, "mixer_index": 10, "normalized_entropy": 0.9780623252441762, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.34262996912002563, "median": 0.3412924110889435, "p05": 0.29556700587272644, "p95": 0.39125555753707886 }, { "label": "completed_group_1", "mean": 0.24457570910453796, "median": 0.24382859468460083, "p05": 0.22120657563209534, "p95": 0.2734297513961792 }, { "label": "current_group_partial", "mean": 0.412794291973114, "median": 0.4151499271392822, "p05": 0.37735897302627563, "p95": 0.43749260902404785 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0074045658111572, "group": 2, "latest_source_mass": 0.3775783181190491, "layer": 6, "max_source_mass": 0.4399106502532959, "mixer_index": 11, "normalized_entropy": 0.9169791528842925, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.4399106502532959, "median": 0.44856905937194824, "p05": 0.22063353657722473, "p95": 0.6359403133392334 }, { "label": "completed_group_1", "mean": 0.18251106142997742, "median": 0.18562838435173035, "p05": 0.1196163073182106, "p95": 0.23045453429222107 }, { "label": "current_group_partial", "mean": 0.3775783181190491, "median": 0.3675365447998047, "p05": 0.23817461729049683, "p95": 0.5557451844215393 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.060497522354126, "group": 2, "latest_source_mass": 0.4025024175643921, "layer": 7, "max_source_mass": 0.4025024175643921, "mixer_index": 12, "normalized_entropy": 0.9653064445872968, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.3786153197288513, "median": 0.3907395303249359, "p05": 0.28039106726646423, "p95": 0.4436405599117279 }, { "label": "completed_group_1", "mean": 0.2188822478055954, "median": 0.21226292848587036, "p05": 0.1885872334241867, "p95": 0.2672030031681061 }, { "label": "current_group_partial", "mean": 0.4025024175643921, "median": 0.39815032482147217, "p05": 0.3533596992492676, "p95": 0.460100919008255 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 0.9941019415855408, "group": 2, "latest_source_mass": 0.4801652431488037, "layer": 7, "max_source_mass": 0.4801652431488037, "mixer_index": 13, "normalized_entropy": 0.9048705824970601, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.32296955585479736, "median": 0.2803739905357361, "p05": 0.1784922033548355, "p95": 0.6440245509147644 }, { "label": "completed_group_1", "mean": 0.19686520099639893, "median": 0.2053825855255127, "p05": 0.09813719242811203, "p95": 0.2504696547985077 }, { "label": "current_group_partial", "mean": 0.4801652431488037, "median": 0.5057140588760376, "p05": 0.2529003322124481, "p95": 0.598870575428009 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0503522157669067, "group": 2, "latest_source_mass": 0.4001518487930298, "layer": 8, "max_source_mass": 0.40130966901779175, "mixer_index": 14, "normalized_entropy": 0.9560717885654542, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.40130966901779175, "median": 0.396759569644928, "p05": 0.3479611873626709, "p95": 0.4733521044254303 }, { "label": "completed_group_1", "mean": 0.19853851199150085, "median": 0.19769755005836487, "p05": 0.17136667668819427, "p95": 0.22788912057876587 }, { "label": "current_group_partial", "mean": 0.4001518487930298, "median": 0.40291696786880493, "p05": 0.3506356179714203, "p95": 0.43941178917884827 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0129402875900269, "group": 2, "latest_source_mass": 0.42161279916763306, "layer": 8, "max_source_mass": 0.42161279916763306, "mixer_index": 15, "normalized_entropy": 0.9220179839951123, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.4093267023563385, "median": 0.4046899676322937, "p05": 0.2895161807537079, "p95": 0.5762131810188293 }, { "label": "completed_group_1", "mean": 0.16906049847602844, "median": 0.16888955235481262, "p05": 0.11233551055192947, "p95": 0.22477403283119202 }, { "label": "current_group_partial", "mean": 0.42161279916763306, "median": 0.4227708578109741, "p05": 0.3131049573421478, "p95": 0.519075334072113 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0774019956588745, "group": 3, "latest_source_mass": 0.3560434877872467, "layer": 9, "max_source_mass": 0.38510820269584656, "mixer_index": 16, "normalized_entropy": 0.9806935592947451, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.38510820269584656, "median": 0.3892287313938141, "p05": 0.2934730052947998, "p95": 0.4718902111053467 }, { "label": "completed_group_1", "mean": 0.25884830951690674, "median": 0.25734007358551025, "p05": 0.22217319905757904, "p95": 0.29551801085472107 }, { "label": "completed_group_2", "mean": 0.3560434877872467, "median": 0.35412120819091797, "p05": 0.29444968700408936, "p95": 0.41908568143844604 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.2740377187728882, "group": 3, "latest_source_mass": 0.3633861243724823, "layer": 9, "max_source_mass": 0.3633861243724823, "mixer_index": 17, "normalized_entropy": 0.9190239493895668, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.29550763964653015, "median": 0.30360478162765503, "p05": 0.16904757916927338, "p95": 0.45146965980529785 }, { "label": "completed_group_1", "mean": 0.09541244804859161, "median": 0.09486871212720871, "p05": 0.061360105872154236, "p95": 0.13037073612213135 }, { "label": "completed_group_2", "mean": 0.24569380283355713, "median": 0.24333015084266663, "p05": 0.1698218435049057, "p95": 0.31589236855506897 }, { "label": "current_group_partial", "mean": 0.3633861243724823, "median": 0.36472851037979126, "p05": 0.2795379161834717, "p95": 0.4399876892566681 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.360302448272705, "group": 3, "latest_source_mass": 0.28569501638412476, "layer": 10, "max_source_mass": 0.3109527826309204, "mixer_index": 18, "normalized_entropy": 0.9812507981160736, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.3109527826309204, "median": 0.32172977924346924, "p05": 0.23520725965499878, "p95": 0.37052470445632935 }, { "label": "completed_group_1", "mean": 0.20984819531440735, "median": 0.20958667993545532, "p05": 0.1891367882490158, "p95": 0.23184512555599213 }, { "label": "completed_group_2", "mean": 0.19350400567054749, "median": 0.19153150916099548, "p05": 0.16607457399368286, "p95": 0.22877082228660583 }, { "label": "current_group_partial", "mean": 0.28569501638412476, "median": 0.28060320019721985, "p05": 0.23824527859687805, "p95": 0.3439165949821472 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.2722704410552979, "group": 3, "latest_source_mass": 0.29912811517715454, "layer": 10, "max_source_mass": 0.35036465525627136, "mixer_index": 19, "normalized_entropy": 0.9177491279900463, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.35036465525627136, "median": 0.36210548877716064, "p05": 0.12807878851890564, "p95": 0.5797606706619263 }, { "label": "completed_group_1", "mean": 0.12631990015506744, "median": 0.1264108121395111, "p05": 0.06977735459804535, "p95": 0.1755298376083374 }, { "label": "completed_group_2", "mean": 0.22418735921382904, "median": 0.22563719749450684, "p05": 0.1176326721906662, "p95": 0.32001155614852905 }, { "label": "current_group_partial", "mean": 0.29912811517715454, "median": 0.28589946031570435, "p05": 0.193109393119812, "p95": 0.44119665026664734 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3558036088943481, "group": 3, "latest_source_mass": 0.314197838306427, "layer": 11, "max_source_mass": 0.314197838306427, "mixer_index": 20, "normalized_entropy": 0.978005571485618, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.298976868391037, "median": 0.2961939573287964, "p05": 0.24950163066387177, "p95": 0.35835936665534973 }, { "label": "completed_group_1", "mean": 0.1941717267036438, "median": 0.1918032318353653, "p05": 0.16809208691120148, "p95": 0.22895419597625732 }, { "label": "completed_group_2", "mean": 0.1926535815000534, "median": 0.19306811690330505, "p05": 0.16983944177627563, "p95": 0.2155935913324356 }, { "label": "current_group_partial", "mean": 0.314197838306427, "median": 0.31541913747787476, "p05": 0.2673022747039795, "p95": 0.35941022634506226 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.2696614265441895, "group": 3, "latest_source_mass": 0.2862822115421295, "layer": 11, "max_source_mass": 0.3667730689048767, "mixer_index": 21, "normalized_entropy": 0.9158671218416545, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.3667730689048767, "median": 0.4067659378051758, "p05": 0.10593366622924805, "p95": 0.5672274827957153 }, { "label": "completed_group_1", "mean": 0.15310388803482056, "median": 0.15050218999385834, "p05": 0.08651046454906464, "p95": 0.22459912300109863 }, { "label": "completed_group_2", "mean": 0.19384080171585083, "median": 0.18732509016990662, "p05": 0.1267009675502777, "p95": 0.281107634305954 }, { "label": "current_group_partial", "mean": 0.2862822115421295, "median": 0.26271265745162964, "p05": 0.17024393379688263, "p95": 0.475044310092926 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3606936931610107, "group": 3, "latest_source_mass": 0.3028184771537781, "layer": 12, "max_source_mass": 0.3028184771537781, "mixer_index": 22, "normalized_entropy": 0.9815330216461395, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.298227459192276, "median": 0.2954302132129669, "p05": 0.25287362933158875, "p95": 0.35405808687210083 }, { "label": "completed_group_1", "mean": 0.18611131608486176, "median": 0.1850002110004425, "p05": 0.15901757776737213, "p95": 0.21725836396217346 }, { "label": "completed_group_2", "mean": 0.21284276247024536, "median": 0.21402311325073242, "p05": 0.18819917738437653, "p95": 0.23418807983398438 }, { "label": "current_group_partial", "mean": 0.3028184771537781, "median": 0.3041122555732727, "p05": 0.25661808252334595, "p95": 0.34458184242248535 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.269316554069519, "group": 3, "latest_source_mass": 0.2653314769268036, "layer": 12, "max_source_mass": 0.4081852436065674, "mixer_index": 23, "normalized_entropy": 0.9156183489371815, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.4081852436065674, "median": 0.4254342317581177, "p05": 0.20765496790409088, "p95": 0.587238073348999 }, { "label": "completed_group_1", "mean": 0.14420127868652344, "median": 0.1438048928976059, "p05": 0.1064920499920845, "p95": 0.1871720254421234 }, { "label": "completed_group_2", "mean": 0.18228203058242798, "median": 0.17507661879062653, "p05": 0.12183455377817154, "p95": 0.26505547761917114 }, { "label": "current_group_partial", "mean": 0.2653314769268036, "median": 0.24924281239509583, "p05": 0.15609969198703766, "p95": 0.4061311185359955 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3636136054992676, "group": 4, "latest_source_mass": 0.3136237859725952, "layer": 13, "max_source_mass": 0.3136237859725952, "mixer_index": 24, "normalized_entropy": 0.9836392931712563, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.27427661418914795, "median": 0.27403637766838074, "p05": 0.22928935289382935, "p95": 0.319773405790329 }, { "label": "completed_group_1", "mean": 0.21341991424560547, "median": 0.20697325468063354, "p05": 0.18552586436271667, "p95": 0.26688331365585327 }, { "label": "completed_group_2", "mean": 0.19867971539497375, "median": 0.197545126080513, "p05": 0.1794913411140442, "p95": 0.222100168466568 }, { "label": "completed_group_3", "mean": 0.3136237859725952, "median": 0.31875866651535034, "p05": 0.24156072735786438, "p95": 0.3695686459541321 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.570773959159851, "group": 4, "latest_source_mass": 0.2294243425130844, "layer": 13, "max_source_mass": 0.2658740282058716, "mixer_index": 25, "normalized_entropy": 0.9759767351225285, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.2658740282058716, "median": 0.2747163474559784, "p05": 0.17927880585193634, "p95": 0.3512483835220337 }, { "label": "completed_group_1", "mean": 0.13419361412525177, "median": 0.13374894857406616, "p05": 0.11072897166013718, "p95": 0.1586270034313202 }, { "label": "completed_group_2", "mean": 0.1907704919576645, "median": 0.19074805080890656, "p05": 0.14699944853782654, "p95": 0.23316305875778198 }, { "label": "completed_group_3", "mean": 0.17973749339580536, "median": 0.17616114020347595, "p05": 0.1363743096590042, "p95": 0.23202598094940186 }, { "label": "current_group_partial", "mean": 0.2294243425130844, "median": 0.22995705902576447, "p05": 0.17975632846355438, "p95": 0.2789157032966614 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.5560038089752197, "group": 4, "latest_source_mass": 0.32468360662460327, "layer": 14, "max_source_mass": 0.32468360662460327, "mixer_index": 26, "normalized_entropy": 0.9667995248241249, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.20477107167243958, "median": 0.206403911113739, "p05": 0.16619175672531128, "p95": 0.2431035190820694 }, { "label": "completed_group_1", "mean": 0.15721669793128967, "median": 0.15658551454544067, "p05": 0.13878680765628815, "p95": 0.17778633534908295 }, { "label": "completed_group_2", "mean": 0.13811248540878296, "median": 0.13743364810943604, "p05": 0.12352626770734787, "p95": 0.15553249418735504 }, { "label": "completed_group_3", "mean": 0.1752161681652069, "median": 0.17309701442718506, "p05": 0.15223677456378937, "p95": 0.20474471151828766 }, { "label": "current_group_partial", "mean": 0.32468360662460327, "median": 0.32497870922088623, "p05": 0.2946181893348694, "p95": 0.35203880071640015 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.5659973621368408, "group": 4, "latest_source_mass": 0.26703086495399475, "layer": 14, "max_source_mass": 0.26703086495399475, "mixer_index": 27, "normalized_entropy": 0.9730088685238187, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.20531430840492249, "median": 0.20229148864746094, "p05": 0.10698921978473663, "p95": 0.3393132984638214 }, { "label": "completed_group_1", "mean": 0.15986156463623047, "median": 0.15952220559120178, "p05": 0.11969029903411865, "p95": 0.20180393755435944 }, { "label": "completed_group_2", "mean": 0.21557100117206573, "median": 0.21654027700424194, "p05": 0.1673085242509842, "p95": 0.2610440254211426 }, { "label": "completed_group_3", "mean": 0.15222224593162537, "median": 0.1510905921459198, "p05": 0.11800260841846466, "p95": 0.18893754482269287 }, { "label": "current_group_partial", "mean": 0.26703086495399475, "median": 0.258539617061615, "p05": 0.19330690801143646, "p95": 0.36521556973457336 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.5552496910095215, "group": 4, "latest_source_mass": 0.31774693727493286, "layer": 15, "max_source_mass": 0.31774693727493286, "mixer_index": 28, "normalized_entropy": 0.9663309649872576, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.2162160575389862, "median": 0.2145705670118332, "p05": 0.17727568745613098, "p95": 0.26878347992897034 }, { "label": "completed_group_1", "mean": 0.17982497811317444, "median": 0.18024154007434845, "p05": 0.15453973412513733, "p95": 0.2030555009841919 }, { "label": "completed_group_2", "mean": 0.1307673156261444, "median": 0.13107559084892273, "p05": 0.11463287472724915, "p95": 0.14630235731601715 }, { "label": "completed_group_3", "mean": 0.1554446816444397, "median": 0.15516987442970276, "p05": 0.13627474009990692, "p95": 0.1754152774810791 }, { "label": "current_group_partial", "mean": 0.31774693727493286, "median": 0.319039523601532, "p05": 0.27606821060180664, "p95": 0.35650834441185 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.5068919658660889, "group": 4, "latest_source_mass": 0.3298327624797821, "layer": 15, "max_source_mass": 0.3298327624797821, "mixer_index": 29, "normalized_entropy": 0.9362846209998111, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.20646697282791138, "median": 0.18775638937950134, "p05": 0.0806753933429718, "p95": 0.4274584949016571 }, { "label": "completed_group_1", "mean": 0.1197461187839508, "median": 0.11989086866378784, "p05": 0.0809398666024208, "p95": 0.15743419528007507 }, { "label": "completed_group_2", "mean": 0.19215428829193115, "median": 0.19620640575885773, "p05": 0.1148109883069992, "p95": 0.2491212636232376 }, { "label": "completed_group_3", "mean": 0.15179988741874695, "median": 0.15212899446487427, "p05": 0.10697861760854721, "p95": 0.19542087614536285 }, { "label": "current_group_partial", "mean": 0.3298327624797821, "median": 0.32837340235710144, "p05": 0.22746729850769043, "p95": 0.4339301884174347 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.5435223579406738, "group": 4, "latest_source_mass": 0.3224271535873413, "layer": 16, "max_source_mass": 0.3224271535873413, "mixer_index": 30, "normalized_entropy": 0.9590443632623663, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.22410975396633148, "median": 0.22760415077209473, "p05": 0.14593973755836487, "p95": 0.30093055963516235 }, { "label": "completed_group_1", "mean": 0.16986176371574402, "median": 0.16963240504264832, "p05": 0.13569067418575287, "p95": 0.20651353895664215 }, { "label": "completed_group_2", "mean": 0.13080403208732605, "median": 0.13075244426727295, "p05": 0.10945598781108856, "p95": 0.15187400579452515 }, { "label": "completed_group_3", "mean": 0.15279728174209595, "median": 0.15036612749099731, "p05": 0.12810495495796204, "p95": 0.18468116223812103 }, { "label": "current_group_partial", "mean": 0.3224271535873413, "median": 0.3164973258972168, "p05": 0.2600978910923004, "p95": 0.40530598163604736 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.4949920177459717, "group": 4, "latest_source_mass": 0.31847020983695984, "layer": 16, "max_source_mass": 0.31847020983695984, "mixer_index": 31, "normalized_entropy": 0.9288907675133354, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.27174079418182373, "median": 0.2585451304912567, "p05": 0.13639721274375916, "p95": 0.4698348045349121 }, { "label": "completed_group_1", "mean": 0.12178683280944824, "median": 0.11714950203895569, "p05": 0.08594125509262085, "p95": 0.1723237782716751 }, { "label": "completed_group_2", "mean": 0.1529826819896698, "median": 0.15223106741905212, "p05": 0.10371473431587219, "p95": 0.203575000166893 }, { "label": "completed_group_3", "mean": 0.1350194811820984, "median": 0.13363957405090332, "p05": 0.08964549005031586, "p95": 0.18468207120895386 }, { "label": "current_group_partial", "mean": 0.31847020983695984, "median": 0.3185216188430786, "p05": 0.21444711089134216, "p95": 0.4301546812057495 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.544264793395996, "group": 5, "latest_source_mass": 0.2861037850379944, "layer": 17, "max_source_mass": 0.2861037850379944, "mixer_index": 32, "normalized_entropy": 0.9595056643474137, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.24016115069389343, "median": 0.23868846893310547, "p05": 0.18435582518577576, "p95": 0.2983453571796417 }, { "label": "completed_group_1", "mean": 0.20730282366275787, "median": 0.1998077780008316, "p05": 0.16255050897598267, "p95": 0.28107208013534546 }, { "label": "completed_group_2", "mean": 0.09692573547363281, "median": 0.0967404842376709, "p05": 0.08098706603050232, "p95": 0.11221462488174438 }, { "label": "completed_group_3", "mean": 0.1695065200328827, "median": 0.1696585714817047, "p05": 0.136396586894989, "p95": 0.20130634307861328 }, { "label": "completed_group_4", "mean": 0.2861037850379944, "median": 0.28883200883865356, "p05": 0.22401469945907593, "p95": 0.3409370183944702 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.686682939529419, "group": 5, "latest_source_mass": 0.12281867861747742, "layer": 17, "max_source_mass": 0.2885604202747345, "mixer_index": 33, "normalized_entropy": 0.9413556721740635, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.16755425930023193, "median": 0.1327776312828064, "p05": 0.0790192037820816, "p95": 0.3667412996292114 }, { "label": "completed_group_1", "mean": 0.10962115228176117, "median": 0.10525335371494293, "p05": 0.07840172946453094, "p95": 0.15405339002609253 }, { "label": "completed_group_2", "mean": 0.1783454418182373, "median": 0.17896884679794312, "p05": 0.10802385956048965, "p95": 0.24048911035060883 }, { "label": "completed_group_3", "mean": 0.13310004770755768, "median": 0.13033035397529602, "p05": 0.0904831737279892, "p95": 0.17934636771678925 }, { "label": "completed_group_4", "mean": 0.2885604202747345, "median": 0.2831825315952301, "p05": 0.16967125236988068, "p95": 0.4189499616622925 }, { "label": "current_group_partial", "mean": 0.12281867861747742, "median": 0.12062528729438782, "p05": 0.07461707293987274, "p95": 0.1782219111919403 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.7070834636688232, "group": 5, "latest_source_mass": 0.2988377809524536, "layer": 18, "max_source_mass": 0.2988377809524536, "mixer_index": 34, "normalized_entropy": 0.9527414214834803, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.181177020072937, "median": 0.18419909477233887, "p05": 0.13624237477779388, "p95": 0.22192169725894928 }, { "label": "completed_group_1", "mean": 0.14557164907455444, "median": 0.144476979970932, "p05": 0.1293315887451172, "p95": 0.16486899554729462 }, { "label": "completed_group_2", "mean": 0.08318156749010086, "median": 0.0820261612534523, "p05": 0.07088497281074524, "p95": 0.0990293100476265 }, { "label": "completed_group_3", "mean": 0.11424209177494049, "median": 0.11267061531543732, "p05": 0.0979941189289093, "p95": 0.1353992223739624 }, { "label": "completed_group_4", "mean": 0.17698991298675537, "median": 0.17597860097885132, "p05": 0.14837387204170227, "p95": 0.21050144731998444 }, { "label": "current_group_partial", "mean": 0.2988377809524536, "median": 0.2989804744720459, "p05": 0.2721370458602905, "p95": 0.32572072744369507 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.7419235706329346, "group": 5, "latest_source_mass": 0.26016175746917725, "layer": 18, "max_source_mass": 0.26016175746917725, "mixer_index": 35, "normalized_entropy": 0.972186055410333, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1532188355922699, "median": 0.1451343297958374, "p05": 0.11514341086149216, "p95": 0.22103679180145264 }, { "label": "completed_group_1", "mean": 0.11420352011919022, "median": 0.11261284351348877, "p05": 0.0928458571434021, "p95": 0.14123988151550293 }, { "label": "completed_group_2", "mean": 0.16597424447536469, "median": 0.1655328869819641, "p05": 0.13871800899505615, "p95": 0.19611573219299316 }, { "label": "completed_group_3", "mean": 0.1250365972518921, "median": 0.12369821965694427, "p05": 0.10375828295946121, "p95": 0.14912742376327515 }, { "label": "completed_group_4", "mean": 0.18140508234500885, "median": 0.1796584278345108, "p05": 0.14511024951934814, "p95": 0.22286373376846313 }, { "label": "current_group_partial", "mean": 0.26016175746917725, "median": 0.26266711950302124, "p05": 0.1889389455318451, "p95": 0.3234381675720215 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.7127439975738525, "group": 5, "latest_source_mass": 0.2907913327217102, "layer": 19, "max_source_mass": 0.2907913327217102, "mixer_index": 36, "normalized_entropy": 0.9559006256078307, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.18731768429279327, "median": 0.19042012095451355, "p05": 0.13879702985286713, "p95": 0.23792000114917755 }, { "label": "completed_group_1", "mean": 0.16715484857559204, "median": 0.1667211651802063, "p05": 0.14574480056762695, "p95": 0.19026139378547668 }, { "label": "completed_group_2", "mean": 0.08496320247650146, "median": 0.08452782034873962, "p05": 0.07410907745361328, "p95": 0.097386933863163 }, { "label": "completed_group_3", "mean": 0.11362075805664062, "median": 0.11248823255300522, "p05": 0.0977015346288681, "p95": 0.13219426572322845 }, { "label": "completed_group_4", "mean": 0.15615218877792358, "median": 0.15540224313735962, "p05": 0.1289147436618805, "p95": 0.18598109483718872 }, { "label": "current_group_partial", "mean": 0.2907913327217102, "median": 0.2900415062904358, "p05": 0.25822532176971436, "p95": 0.32475030422210693 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.6076834201812744, "group": 5, "latest_source_mass": 0.3738824725151062, "layer": 19, "max_source_mass": 0.3738824725151062, "mixer_index": 37, "normalized_entropy": 0.8972652009334232, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.15032559633255005, "median": 0.12544110417366028, "p05": 0.06976518034934998, "p95": 0.35840943455696106 }, { "label": "completed_group_1", "mean": 0.08624927699565887, "median": 0.08602266013622284, "p05": 0.061998721212148666, "p95": 0.11138717085123062 }, { "label": "completed_group_2", "mean": 0.15736976265907288, "median": 0.15802210569381714, "p05": 0.10121746361255646, "p95": 0.20687000453472137 }, { "label": "completed_group_3", "mean": 0.08584291487932205, "median": 0.0827617198228836, "p05": 0.060561805963516235, "p95": 0.11351694166660309 }, { "label": "completed_group_4", "mean": 0.14632999897003174, "median": 0.1426895558834076, "p05": 0.10106660425662994, "p95": 0.20179018378257751 }, { "label": "current_group_partial", "mean": 0.3738824725151062, "median": 0.3799295425415039, "p05": 0.23631010949611664, "p95": 0.4884527325630188 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.709290862083435, "group": 5, "latest_source_mass": 0.2901708781719208, "layer": 20, "max_source_mass": 0.2901708781719208, "mixer_index": 38, "normalized_entropy": 0.9539733939957076, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.18991519510746002, "median": 0.1924753338098526, "p05": 0.14201581478118896, "p95": 0.23709386587142944 }, { "label": "completed_group_1", "mean": 0.17091524600982666, "median": 0.17006167769432068, "p05": 0.15297070145606995, "p95": 0.19153237342834473 }, { "label": "completed_group_2", "mean": 0.07822999358177185, "median": 0.07778562605381012, "p05": 0.06707640737295151, "p95": 0.09089364111423492 }, { "label": "completed_group_3", "mean": 0.116765096783638, "median": 0.11516137421131134, "p05": 0.10162825882434845, "p95": 0.13653048872947693 }, { "label": "completed_group_4", "mean": 0.1540035903453827, "median": 0.15287795662879944, "p05": 0.12593276798725128, "p95": 0.18590238690376282 }, { "label": "current_group_partial", "mean": 0.2901708781719208, "median": 0.29104921221733093, "p05": 0.25643545389175415, "p95": 0.3205650746822357 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.6690887212753296, "group": 5, "latest_source_mass": 0.3342471718788147, "layer": 20, "max_source_mass": 0.3342471718788147, "mixer_index": 39, "normalized_entropy": 0.9315361520005944, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.19029226899147034, "median": 0.19644704461097717, "p05": 0.09456793963909149, "p95": 0.30468082427978516 }, { "label": "completed_group_1", "mean": 0.1100241169333458, "median": 0.10808776319026947, "p05": 0.09035106748342514, "p95": 0.1359245330095291 }, { "label": "completed_group_2", "mean": 0.11471512168645859, "median": 0.11494918912649155, "p05": 0.08799737691879272, "p95": 0.14024731516838074 }, { "label": "completed_group_3", "mean": 0.09999909996986389, "median": 0.09682698547840118, "p05": 0.07921088486909866, "p95": 0.1264391839504242 }, { "label": "completed_group_4", "mean": 0.1507222205400467, "median": 0.1476232260465622, "p05": 0.11397356539964676, "p95": 0.19991227984428406 }, { "label": "current_group_partial", "mean": 0.3342471718788147, "median": 0.33802270889282227, "p05": 0.2548266053199768, "p95": 0.40130624175071716 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.7458693981170654, "group": 6, "latest_source_mass": 0.2361394464969635, "layer": 21, "max_source_mass": 0.2361394464969635, "mixer_index": 40, "normalized_entropy": 0.9743882636597644, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.20528709888458252, "median": 0.2140858769416809, "p05": 0.13168928027153015, "p95": 0.27098023891448975 }, { "label": "completed_group_1", "mean": 0.1722964197397232, "median": 0.17238952219486237, "p05": 0.15053534507751465, "p95": 0.19365043938159943 }, { "label": "completed_group_2", "mean": 0.10435444861650467, "median": 0.10361101478338242, "p05": 0.08888213336467743, "p95": 0.12212536484003067 }, { "label": "completed_group_3", "mean": 0.12075909972190857, "median": 0.1180175170302391, "p05": 0.10358668118715286, "p95": 0.14458929002285004 }, { "label": "completed_group_4", "mean": 0.16116347908973694, "median": 0.15841388702392578, "p05": 0.1337081342935562, "p95": 0.19791515171527863 }, { "label": "completed_group_5", "mean": 0.2361394464969635, "median": 0.23709020018577576, "p05": 0.20446214079856873, "p95": 0.26509594917297363 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.4612125158309937, "group": 6, "latest_source_mass": 0.5543455481529236, "layer": 21, "max_source_mass": 0.5543455481529236, "mixer_index": 41, "normalized_entropy": 0.7509146897354808, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.08934466540813446, "median": 0.07911523431539536, "p05": 0.04488275945186615, "p95": 0.1737232506275177 }, { "label": "completed_group_1", "mean": 0.054379627108573914, "median": 0.05344422906637192, "p05": 0.04221750423312187, "p95": 0.06924191862344742 }, { "label": "completed_group_2", "mean": 0.07567165046930313, "median": 0.07438549399375916, "p05": 0.05811355262994766, "p95": 0.09632256627082825 }, { "label": "completed_group_3", "mean": 0.05758609622716904, "median": 0.05612001568078995, "p05": 0.044792283326387405, "p95": 0.07347827404737473 }, { "label": "completed_group_4", "mean": 0.07884950935840607, "median": 0.07757881283760071, "p05": 0.05758974328637123, "p95": 0.10452556610107422 }, { "label": "completed_group_5", "mean": 0.08982288837432861, "median": 0.08912864327430725, "p05": 0.0663854256272316, "p95": 0.11604885756969452 }, { "label": "current_group_partial", "mean": 0.5543455481529236, "median": 0.5596089363098145, "p05": 0.45455610752105713, "p95": 0.6367484331130981 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.7241814136505127, "group": 6, "latest_source_mass": 0.39484071731567383, "layer": 22, "max_source_mass": 0.39484071731567383, "mixer_index": 42, "normalized_entropy": 0.886053970419732, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.13360010087490082, "median": 0.14072942733764648, "p05": 0.07699814438819885, "p95": 0.17506597936153412 }, { "label": "completed_group_1", "mean": 0.10609637200832367, "median": 0.10586297512054443, "p05": 0.08648421615362167, "p95": 0.12602579593658447 }, { "label": "completed_group_2", "mean": 0.06363911926746368, "median": 0.06324908882379532, "p05": 0.054980404675006866, "p95": 0.07316673547029495 }, { "label": "completed_group_3", "mean": 0.07426095008850098, "median": 0.07362888753414154, "p05": 0.06318790465593338, "p95": 0.08665015548467636 }, { "label": "completed_group_4", "mean": 0.09383076429367065, "median": 0.09294568747282028, "p05": 0.07638302445411682, "p95": 0.11445100605487823 }, { "label": "completed_group_5", "mean": 0.13373197615146637, "median": 0.13327741622924805, "p05": 0.11399447172880173, "p95": 0.15641780197620392 }, { "label": "current_group_partial", "mean": 0.39484071731567383, "median": 0.3879151940345764, "p05": 0.3302172124385834, "p95": 0.4852263331413269 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.3979300260543823, "group": 6, "latest_source_mass": 0.5830056071281433, "layer": 22, "max_source_mass": 0.5830056071281433, "mixer_index": 43, "normalized_entropy": 0.7183939231382495, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.08959861099720001, "median": 0.07544538378715515, "p05": 0.04165133833885193, "p95": 0.20099139213562012 }, { "label": "completed_group_1", "mean": 0.05414731800556183, "median": 0.05336771905422211, "p05": 0.04069708660244942, "p95": 0.07046990841627121 }, { "label": "completed_group_2", "mean": 0.07668881118297577, "median": 0.07607124000787735, "p05": 0.0555989071726799, "p95": 0.09980960935354233 }, { "label": "completed_group_3", "mean": 0.05603286623954773, "median": 0.054573021829128265, "p05": 0.04162490367889404, "p95": 0.0730118378996849 }, { "label": "completed_group_4", "mean": 0.06979228556156158, "median": 0.06884932518005371, "p05": 0.04576495662331581, "p95": 0.09806846082210541 }, { "label": "completed_group_5", "mean": 0.07073447108268738, "median": 0.07023533433675766, "p05": 0.04956750199198723, "p95": 0.09379896521568298 }, { "label": "current_group_partial", "mean": 0.5830056071281433, "median": 0.5862616896629333, "p05": 0.4658474922180176, "p95": 0.6849814057350159 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.7459560632705688, "group": 6, "latest_source_mass": 0.37691548466682434, "layer": 23, "max_source_mass": 0.37691548466682434, "mixer_index": 44, "normalized_entropy": 0.897243926765161, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.14318783581256866, "median": 0.1493408977985382, "p05": 0.08292607218027115, "p95": 0.19464610517024994 }, { "label": "completed_group_1", "mean": 0.1094653457403183, "median": 0.10858799517154694, "p05": 0.09432064741849899, "p95": 0.1279931515455246 }, { "label": "completed_group_2", "mean": 0.06349228322505951, "median": 0.06325754523277283, "p05": 0.05439673364162445, "p95": 0.0737837627530098 }, { "label": "completed_group_3", "mean": 0.07859592139720917, "median": 0.0775163471698761, "p05": 0.06603655964136124, "p95": 0.09357087314128876 }, { "label": "completed_group_4", "mean": 0.09567482024431229, "median": 0.09468592703342438, "p05": 0.07671141624450684, "p95": 0.11774884164333344 }, { "label": "completed_group_5", "mean": 0.13266831636428833, "median": 0.13199329376220703, "p05": 0.11244359612464905, "p95": 0.15504513680934906 }, { "label": "current_group_partial", "mean": 0.37691548466682434, "median": 0.37180638313293457, "p05": 0.31207239627838135, "p95": 0.4595233201980591 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.4055393934249878, "group": 6, "latest_source_mass": 0.5772827863693237, "layer": 23, "max_source_mass": 0.5772827863693237, "mixer_index": 45, "normalized_entropy": 0.7223043644164862, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.10154955089092255, "median": 0.08766824007034302, "p05": 0.04901552200317383, "p95": 0.20287738740444183 }, { "label": "completed_group_1", "mean": 0.05619276314973831, "median": 0.055477917194366455, "p05": 0.041608311235904694, "p95": 0.07371112704277039 }, { "label": "completed_group_2", "mean": 0.07791627198457718, "median": 0.07736755907535553, "p05": 0.055920060724020004, "p95": 0.10158214718103409 }, { "label": "completed_group_3", "mean": 0.053915880620479584, "median": 0.05215863138437271, "p05": 0.03798118978738785, "p95": 0.07156119495630264 }, { "label": "completed_group_4", "mean": 0.06398355215787888, "median": 0.06229078769683838, "p05": 0.040616974234580994, "p95": 0.09270931780338287 }, { "label": "completed_group_5", "mean": 0.06915920227766037, "median": 0.0684593990445137, "p05": 0.046986356377601624, "p95": 0.09408729523420334 }, { "label": "current_group_partial", "mean": 0.5772827863693237, "median": 0.5824689269065857, "p05": 0.4518115520477295, "p95": 0.6926202178001404 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.7853620052337646, "group": 6, "latest_source_mass": 0.3407441973686218, "layer": 24, "max_source_mass": 0.3407441973686218, "mixer_index": 46, "normalized_entropy": 0.9174945750195659, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.15284113585948944, "median": 0.1593906283378601, "p05": 0.08678612112998962, "p95": 0.211342453956604 }, { "label": "completed_group_1", "mean": 0.12206703424453735, "median": 0.12153112143278122, "p05": 0.1060115247964859, "p95": 0.1399766057729721 }, { "label": "completed_group_2", "mean": 0.06508444249629974, "median": 0.06482555717229843, "p05": 0.055350616574287415, "p95": 0.07557765394449234 }, { "label": "completed_group_3", "mean": 0.08347313106060028, "median": 0.08190013468265533, "p05": 0.06966532766819, "p95": 0.10050469636917114 }, { "label": "completed_group_4", "mean": 0.09811970591545105, "median": 0.09664031863212585, "p05": 0.07688035070896149, "p95": 0.12369741499423981 }, { "label": "completed_group_5", "mean": 0.13767032325267792, "median": 0.1373325139284134, "p05": 0.11642646044492722, "p95": 0.1606694608926773 }, { "label": "current_group_partial", "mean": 0.3407441973686218, "median": 0.33805108070373535, "p05": 0.271301805973053, "p95": 0.41781002283096313 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.509842872619629, "group": 6, "latest_source_mass": 0.5227814316749573, "layer": 24, "max_source_mass": 0.5227814316749573, "mixer_index": 47, "normalized_entropy": 0.7759057494780099, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.11788781732320786, "median": 0.10097194463014603, "p05": 0.06265684217214584, "p95": 0.2408325970172882 }, { "label": "completed_group_1", "mean": 0.06822571158409119, "median": 0.0673636943101883, "p05": 0.05209728330373764, "p95": 0.08648915588855743 }, { "label": "completed_group_2", "mean": 0.08556114137172699, "median": 0.08463940769433975, "p05": 0.06234566867351532, "p95": 0.1125786080956459 }, { "label": "completed_group_3", "mean": 0.06067090481519699, "median": 0.058974385261535645, "p05": 0.043267179280519485, "p95": 0.07997901737689972 }, { "label": "completed_group_4", "mean": 0.07228855788707733, "median": 0.06972275674343109, "p05": 0.04689513146877289, "p95": 0.106512151658535 }, { "label": "completed_group_5", "mean": 0.07258446514606476, "median": 0.07082539796829224, "p05": 0.050949547439813614, "p95": 0.09970614314079285 }, { "label": "current_group_partial", "mean": 0.5227814316749573, "median": 0.5290273427963257, "p05": 0.38773614168167114, "p95": 0.6392589807510376 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.839367389678955, "group": 7, "latest_source_mass": 0.29583215713500977, "layer": 25, "max_source_mass": 0.29583215713500977, "mixer_index": 48, "normalized_entropy": 0.9452478525649903, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.15906387567520142, "median": 0.15846745669841766, "p05": 0.09519976377487183, "p95": 0.24517810344696045 }, { "label": "completed_group_1", "mean": 0.13139380514621735, "median": 0.1308426856994629, "p05": 0.11478274315595627, "p95": 0.15000829100608826 }, { "label": "completed_group_2", "mean": 0.07717745751142502, "median": 0.07715500891208649, "p05": 0.06616044044494629, "p95": 0.08791667222976685 }, { "label": "completed_group_3", "mean": 0.09607303142547607, "median": 0.09436552226543427, "p05": 0.08152313530445099, "p95": 0.11384639143943787 }, { "label": "completed_group_4", "mean": 0.1107850894331932, "median": 0.1089654266834259, "p05": 0.08814431726932526, "p95": 0.139452263712883 }, { "label": "completed_group_5", "mean": 0.12967455387115479, "median": 0.1299237608909607, "p05": 0.10832332074642181, "p95": 0.15138982236385345 }, { "label": "completed_group_6", "mean": 0.29583215713500977, "median": 0.29647016525268555, "p05": 0.2276897132396698, "p95": 0.3604946434497833 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.5841834545135498, "group": 7, "latest_source_mass": 0.40767741203308105, "layer": 25, "max_source_mass": 0.40767741203308105, "mixer_index": 49, "normalized_entropy": 0.7618312045616817, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.07896879315376282, "median": 0.06809917092323303, "p05": 0.025932887569069862, "p95": 0.19368338584899902 }, { "label": "completed_group_1", "mean": 0.048752427101135254, "median": 0.04767637699842453, "p05": 0.034443892538547516, "p95": 0.0667153000831604 }, { "label": "completed_group_2", "mean": 0.055707164108753204, "median": 0.0549345538020134, "p05": 0.038696639239788055, "p95": 0.0750277042388916 }, { "label": "completed_group_3", "mean": 0.04139826446771622, "median": 0.03878612071275711, "p05": 0.026673054322600365, "p95": 0.05746251344680786 }, { "label": "completed_group_4", "mean": 0.04449018836021423, "median": 0.04196460545063019, "p05": 0.026162121444940567, "p95": 0.07164917886257172 }, { "label": "completed_group_5", "mean": 0.04285799711942673, "median": 0.04109475016593933, "p05": 0.029393533244729042, "p95": 0.06277710199356079 }, { "label": "completed_group_6", "mean": 0.2801477909088135, "median": 0.2763082981109619, "p05": 0.1455708146095276, "p95": 0.4254714846611023 }, { "label": "current_group_partial", "mean": 0.40767741203308105, "median": 0.4120897948741913, "p05": 0.21399420499801636, "p95": 0.5853031277656555 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.8125314712524414, "group": 7, "latest_source_mass": 0.3700707256793976, "layer": 26, "max_source_mass": 0.3700707256793976, "mixer_index": 50, "normalized_entropy": 0.8716433883436914, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.10889513790607452, "median": 0.11333495378494263, "p05": 0.04850132763385773, "p95": 0.1620434820652008 }, { "label": "completed_group_1", "mean": 0.08656036853790283, "median": 0.08640051633119583, "p05": 0.0650845319032669, "p95": 0.1084962785243988 }, { "label": "completed_group_2", "mean": 0.048123255372047424, "median": 0.04800213500857353, "p05": 0.03576180338859558, "p95": 0.06107594817876816 }, { "label": "completed_group_3", "mean": 0.06561170518398285, "median": 0.06379580497741699, "p05": 0.050968848168849945, "p95": 0.08332888036966324 }, { "label": "completed_group_4", "mean": 0.06917259097099304, "median": 0.06724178791046143, "p05": 0.050207968801259995, "p95": 0.09373342245817184 }, { "label": "completed_group_5", "mean": 0.08286689221858978, "median": 0.08183546364307404, "p05": 0.06535022705793381, "p95": 0.10340933501720428 }, { "label": "completed_group_6", "mean": 0.16869932413101196, "median": 0.16509854793548584, "p05": 0.11059704422950745, "p95": 0.23654375970363617 }, { "label": "current_group_partial", "mean": 0.3700707256793976, "median": 0.3694133758544922, "p05": 0.23441238701343536, "p95": 0.5025027990341187 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.50089693069458, "group": 7, "latest_source_mass": 0.49096518754959106, "layer": 26, "max_source_mass": 0.49096518754959106, "mixer_index": 51, "normalized_entropy": 0.7217788529328457, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.07974998652935028, "median": 0.07402504235506058, "p05": 0.04579866677522659, "p95": 0.1359037309885025 }, { "label": "completed_group_1", "mean": 0.04166398197412491, "median": 0.04038450866937637, "p05": 0.027992289513349533, "p95": 0.05969971418380737 }, { "label": "completed_group_2", "mean": 0.04607272148132324, "median": 0.044438350945711136, "p05": 0.030038591474294662, "p95": 0.06808309257030487 }, { "label": "completed_group_3", "mean": 0.03586764633655548, "median": 0.03317485377192497, "p05": 0.02203454077243805, "p95": 0.05572832003235817 }, { "label": "completed_group_4", "mean": 0.04223340004682541, "median": 0.03930673003196716, "p05": 0.02465122565627098, "p95": 0.06968094408512115 }, { "label": "completed_group_5", "mean": 0.04095568507909775, "median": 0.03919431194663048, "p05": 0.026239857077598572, "p95": 0.06058088690042496 }, { "label": "completed_group_6", "mean": 0.22249138355255127, "median": 0.2125401496887207, "p05": 0.10647299885749817, "p95": 0.3711881935596466 }, { "label": "current_group_partial", "mean": 0.49096518754959106, "median": 0.4988659620285034, "p05": 0.2878011167049408, "p95": 0.6676807403564453 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.8059929609298706, "group": 7, "latest_source_mass": 0.37139976024627686, "layer": 27, "max_source_mass": 0.37139976024627686, "mixer_index": 52, "normalized_entropy": 0.8684990295379666, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.10711923241615295, "median": 0.10972198843955994, "p05": 0.04896187782287598, "p95": 0.16201043128967285 }, { "label": "completed_group_1", "mean": 0.07651707530021667, "median": 0.07666032761335373, "p05": 0.05141128972172737, "p95": 0.10152198374271393 }, { "label": "completed_group_2", "mean": 0.058284737169742584, "median": 0.05812416970729828, "p05": 0.039273619651794434, "p95": 0.07745087146759033 }, { "label": "completed_group_3", "mean": 0.06609587371349335, "median": 0.06383530795574188, "p05": 0.04481930285692215, "p95": 0.09336265176534653 }, { "label": "completed_group_4", "mean": 0.07139429450035095, "median": 0.0694514587521553, "p05": 0.0485839769244194, "p95": 0.10034126043319702 }, { "label": "completed_group_5", "mean": 0.07493151724338531, "median": 0.07319007813930511, "p05": 0.051876723766326904, "p95": 0.10402432829141617 }, { "label": "completed_group_6", "mean": 0.17425751686096191, "median": 0.16937679052352905, "p05": 0.10449051856994629, "p95": 0.2624811828136444 }, { "label": "current_group_partial", "mean": 0.37139976024627686, "median": 0.3732905089855194, "p05": 0.23830299079418182, "p95": 0.5014542937278748 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.5622732639312744, "group": 7, "latest_source_mass": 0.4768889844417572, "layer": 27, "max_source_mass": 0.4768889844417572, "mixer_index": 53, "normalized_entropy": 0.7512946301290215, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.08775827288627625, "median": 0.08504067361354828, "p05": 0.0428270623087883, "p95": 0.144285649061203 }, { "label": "completed_group_1", "mean": 0.047765426337718964, "median": 0.04646281152963638, "p05": 0.031814731657505035, "p95": 0.06807165592908859 }, { "label": "completed_group_2", "mean": 0.053928617388010025, "median": 0.052262187004089355, "p05": 0.035437777638435364, "p95": 0.0784272700548172 }, { "label": "completed_group_3", "mean": 0.04293380305171013, "median": 0.03996016085147858, "p05": 0.02556057646870613, "p95": 0.06623870134353638 }, { "label": "completed_group_4", "mean": 0.04174989461898804, "median": 0.03903091698884964, "p05": 0.025013400241732597, "p95": 0.06869364529848099 }, { "label": "completed_group_5", "mean": 0.04734005779027939, "median": 0.04609068110585213, "p05": 0.029884062707424164, "p95": 0.06911227852106094 }, { "label": "completed_group_6", "mean": 0.20163494348526, "median": 0.19222964346408844, "p05": 0.09963390231132507, "p95": 0.3307305574417114 }, { "label": "current_group_partial", "mean": 0.4768889844417572, "median": 0.48383551836013794, "p05": 0.276212215423584, "p95": 0.6487575173377991 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.8757455348968506, "group": 7, "latest_source_mass": 0.3109493851661682, "layer": 28, "max_source_mass": 0.3109493851661682, "mixer_index": 54, "normalized_entropy": 0.9020429270551009, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.11563669890165329, "median": 0.12087491154670715, "p05": 0.05733206868171692, "p95": 0.1688953936100006 }, { "label": "completed_group_1", "mean": 0.08108697831630707, "median": 0.08185897022485733, "p05": 0.05175650864839554, "p95": 0.10753051191568375 }, { "label": "completed_group_2", "mean": 0.07280858606100082, "median": 0.07357072830200195, "p05": 0.04870978742837906, "p95": 0.0948689877986908 }, { "label": "completed_group_3", "mean": 0.07553227245807648, "median": 0.07362619787454605, "p05": 0.056317005306482315, "p95": 0.09756095707416534 }, { "label": "completed_group_4", "mean": 0.07606014609336853, "median": 0.07456053793430328, "p05": 0.05528895929455757, "p95": 0.1022825762629509 }, { "label": "completed_group_5", "mean": 0.06935927271842957, "median": 0.06843917071819305, "p05": 0.04978285729885101, "p95": 0.09167985618114471 }, { "label": "completed_group_6", "mean": 0.19856664538383484, "median": 0.19508783519268036, "p05": 0.1281973123550415, "p95": 0.27796900272369385 }, { "label": "current_group_partial", "mean": 0.3109493851661682, "median": 0.3105033040046692, "p05": 0.18172834813594818, "p95": 0.43916669487953186 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.6924852132797241, "group": 7, "latest_source_mass": 0.3315581679344177, "layer": 28, "max_source_mass": 0.3315581679344177, "mixer_index": 55, "normalized_entropy": 0.8139133413255193, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.08638577908277512, "median": 0.058608733117580414, "p05": 0.026544082909822464, "p95": 0.2740764319896698 }, { "label": "completed_group_1", "mean": 0.059912845492362976, "median": 0.05901291221380234, "p05": 0.03629481792449951, "p95": 0.08707985281944275 }, { "label": "completed_group_2", "mean": 0.06609142571687698, "median": 0.0647939145565033, "p05": 0.038691695779561996, "p95": 0.09703366458415985 }, { "label": "completed_group_3", "mean": 0.056745611131191254, "median": 0.05418796092271805, "p05": 0.032174739986658096, "p95": 0.08370660990476608 }, { "label": "completed_group_4", "mean": 0.06277773529291153, "median": 0.0600711889564991, "p05": 0.03501531109213829, "p95": 0.10094858705997467 }, { "label": "completed_group_5", "mean": 0.05352145433425903, "median": 0.05179818347096443, "p05": 0.032271772623062134, "p95": 0.08084158599376678 }, { "label": "completed_group_6", "mean": 0.2830069661140442, "median": 0.2759752869606018, "p05": 0.12981083989143372, "p95": 0.46204638481140137 }, { "label": "current_group_partial", "mean": 0.3315581679344177, "median": 0.3204721510410309, "p05": 0.13446375727653503, "p95": 0.5667697787284851 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.8980522155761719, "group": 8, "latest_source_mass": 0.20123262703418732, "layer": 29, "max_source_mass": 0.26602888107299805, "mixer_index": 56, "normalized_entropy": 0.9127701729200177, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.11603894829750061, "median": 0.09991352260112762, "p05": 0.030035395175218582, "p95": 0.2941743731498718 }, { "label": "completed_group_1", "mean": 0.08804173022508621, "median": 0.08880250155925751, "p05": 0.05664174258708954, "p95": 0.11779069900512695 }, { "label": "completed_group_2", "mean": 0.07975024729967117, "median": 0.07984748482704163, "p05": 0.05428915470838547, "p95": 0.10499341040849686 }, { "label": "completed_group_3", "mean": 0.09027000516653061, "median": 0.08898856490850449, "p05": 0.06189651042222977, "p95": 0.11996553838253021 }, { "label": "completed_group_4", "mean": 0.08430293202400208, "median": 0.0819794088602066, "p05": 0.05464346706867218, "p95": 0.12159879505634308 }, { "label": "completed_group_5", "mean": 0.07433462142944336, "median": 0.07287181913852692, "p05": 0.052520088851451874, "p95": 0.1010315865278244 }, { "label": "completed_group_6", "mean": 0.26602888107299805, "median": 0.2639757990837097, "p05": 0.15474572777748108, "p95": 0.3846370279788971 }, { "label": "completed_group_7", "mean": 0.20123262703418732, "median": 0.18140269815921783, "p05": 0.08419936150312424, "p95": 0.3782624900341034 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.7512003183364868, "group": 8, "latest_source_mass": 0.33482813835144043, "layer": 29, "max_source_mass": 0.33482813835144043, "mixer_index": 57, "normalized_entropy": 0.7970056117156376, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.07744461297988892, "median": 0.06970229744911194, "p05": 0.014401955530047417, "p95": 0.17144526541233063 }, { "label": "completed_group_1", "mean": 0.046086348593235016, "median": 0.04394831880927086, "p05": 0.028066720813512802, "p95": 0.0718703418970108 }, { "label": "completed_group_2", "mean": 0.05025433003902435, "median": 0.04861236736178398, "p05": 0.029644077643752098, "p95": 0.07761462777853012 }, { "label": "completed_group_3", "mean": 0.03748488053679466, "median": 0.03434944897890091, "p05": 0.020774798467755318, "p95": 0.05938909202814102 }, { "label": "completed_group_4", "mean": 0.04005061462521553, "median": 0.037498839199543, "p05": 0.019541850313544273, "p95": 0.06922487914562225 }, { "label": "completed_group_5", "mean": 0.041623204946517944, "median": 0.039552345871925354, "p05": 0.022081367671489716, "p95": 0.06821337342262268 }, { "label": "completed_group_6", "mean": 0.198423832654953, "median": 0.18318486213684082, "p05": 0.07548617571592331, "p95": 0.3721975088119507 }, { "label": "completed_group_7", "mean": 0.17380405962467194, "median": 0.1514919549226761, "p05": 0.05777408182621002, "p95": 0.36326974630355835 }, { "label": "current_group_partial", "mean": 0.33482813835144043, "median": 0.33602219820022583, "p05": 0.0795319601893425, "p95": 0.5979493856430054 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.0160694122314453, "group": 8, "latest_source_mass": 0.21711516380310059, "layer": 30, "max_source_mass": 0.21711516380310059, "mixer_index": 58, "normalized_entropy": 0.9175527313077866, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.09385271370410919, "median": 0.09211117774248123, "p05": 0.032377734780311584, "p95": 0.16496515274047852 }, { "label": "completed_group_1", "mean": 0.06208505108952522, "median": 0.06140740588307381, "p05": 0.0407925546169281, "p95": 0.0863042101264 }, { "label": "completed_group_2", "mean": 0.06560768187046051, "median": 0.06516212224960327, "p05": 0.041897669434547424, "p95": 0.09085705131292343 }, { "label": "completed_group_3", "mean": 0.07029052078723907, "median": 0.06780634075403214, "p05": 0.04846132919192314, "p95": 0.09917552024126053 }, { "label": "completed_group_4", "mean": 0.06935097277164459, "median": 0.06708167493343353, "p05": 0.04668639227747917, "p95": 0.09815650433301926 }, { "label": "completed_group_5", "mean": 0.056832700967788696, "median": 0.05595892667770386, "p05": 0.038822539150714874, "p95": 0.0771557092666626 }, { "label": "completed_group_6", "mean": 0.19425815343856812, "median": 0.19145764410495758, "p05": 0.12249615788459778, "p95": 0.27496030926704407 }, { "label": "completed_group_7", "mean": 0.17060703039169312, "median": 0.16317546367645264, "p05": 0.09239593893289566, "p95": 0.2726203501224518 }, { "label": "current_group_partial", "mean": 0.21711516380310059, "median": 0.21198657155036926, "p05": 0.11019522696733475, "p95": 0.3391069769859314 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 1.7837426662445068, "group": 8, "latest_source_mass": 0.29026859998703003, "layer": 30, "max_source_mass": 0.29026859998703003, "mixer_index": 59, "normalized_entropy": 0.8118162725118464, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.08836327493190765, "median": 0.07904411852359772, "p05": 0.02631576545536518, "p95": 0.17534467577934265 }, { "label": "completed_group_1", "mean": 0.04953558370471001, "median": 0.04658038914203644, "p05": 0.025181813165545464, "p95": 0.08318150043487549 }, { "label": "completed_group_2", "mean": 0.04813014343380928, "median": 0.045740917325019836, "p05": 0.026714935898780823, "p95": 0.07843323051929474 }, { "label": "completed_group_3", "mean": 0.0373263955116272, "median": 0.03401809185743332, "p05": 0.01990434154868126, "p95": 0.06290259957313538 }, { "label": "completed_group_4", "mean": 0.037015400826931, "median": 0.03405017405748367, "p05": 0.018646355718374252, "p95": 0.06544196605682373 }, { "label": "completed_group_5", "mean": 0.03390160948038101, "median": 0.03198609501123428, "p05": 0.018189728260040283, "p95": 0.05570102855563164 }, { "label": "completed_group_6", "mean": 0.1840541511774063, "median": 0.17327937483787537, "p05": 0.07966600358486176, "p95": 0.32982927560806274 }, { "label": "completed_group_7", "mean": 0.23140482604503632, "median": 0.22406929731369019, "p05": 0.08831030130386353, "p95": 0.40350067615509033 }, { "label": "current_group_partial", "mean": 0.29026859998703003, "median": 0.2745037376880646, "p05": 0.10280418395996094, "p95": 0.5399085879325867 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 1.9973840713500977, "group": 8, "latest_source_mass": 0.19101998209953308, "layer": 31, "max_source_mass": 0.20831164717674255, "mixer_index": 60, "normalized_entropy": 0.9090486661912383, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.10261696577072144, "median": 0.10389091819524765, "p05": 0.01371697522699833, "p95": 0.21163305640220642 }, { "label": "completed_group_1", "mean": 0.06039813905954361, "median": 0.06085003539919853, "p05": 0.026107093319296837, "p95": 0.09348798543214798 }, { "label": "completed_group_2", "mean": 0.06671924144029617, "median": 0.06656201928853989, "p05": 0.034684643149375916, "p95": 0.09819839894771576 }, { "label": "completed_group_3", "mean": 0.08377155661582947, "median": 0.0804532840847969, "p05": 0.05448959022760391, "p95": 0.12039553374052048 }, { "label": "completed_group_4", "mean": 0.07002593576908112, "median": 0.06737451255321503, "p05": 0.044841229915618896, "p95": 0.10655929893255234 }, { "label": "completed_group_5", "mean": 0.05768897384405136, "median": 0.056077782064676285, "p05": 0.039495617151260376, "p95": 0.08242568373680115 }, { "label": "completed_group_6", "mean": 0.20831164717674255, "median": 0.2021285891532898, "p05": 0.11652229726314545, "p95": 0.31706833839416504 }, { "label": "completed_group_7", "mean": 0.1594475358724594, "median": 0.1546279788017273, "p05": 0.06593266129493713, "p95": 0.2735573351383209 }, { "label": "current_group_partial", "mean": 0.19101998209953308, "median": 0.1713225245475769, "p05": 0.06408411264419556, "p95": 0.3850628137588501 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 1.969987154006958, "group": 8, "latest_source_mass": 0.16864418983459473, "layer": 31, "max_source_mass": 0.24503237009048462, "mixer_index": 61, "normalized_entropy": 0.8965797917640489, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.08120635151863098, "median": 0.061164144426584244, "p05": 0.0259944386780262, "p95": 0.23622018098831177 }, { "label": "completed_group_1", "mean": 0.07894632965326309, "median": 0.07658770680427551, "p05": 0.0496862456202507, "p95": 0.11596276611089706 }, { "label": "completed_group_2", "mean": 0.08615730702877045, "median": 0.08467075228691101, "p05": 0.050409119576215744, "p95": 0.12811316549777985 }, { "label": "completed_group_3", "mean": 0.07609779387712479, "median": 0.0722871869802475, "p05": 0.04099677503108978, "p95": 0.11950768530368805 }, { "label": "completed_group_4", "mean": 0.058728985488414764, "median": 0.056080836802721024, "p05": 0.03323862701654434, "p95": 0.09344407916069031 }, { "label": "completed_group_5", "mean": 0.05076971277594566, "median": 0.04946017265319824, "p05": 0.03047419898211956, "p95": 0.07563400268554688 }, { "label": "completed_group_6", "mean": 0.24503237009048462, "median": 0.24110376834869385, "p05": 0.10613486170768738, "p95": 0.3991503119468689 }, { "label": "completed_group_7", "mean": 0.1544169783592224, "median": 0.1400495171546936, "p05": 0.06199025735259056, "p95": 0.2995939254760742 }, { "label": "current_group_partial", "mean": 0.16864418983459473, "median": 0.15222924947738647, "p05": 0.0543956533074379, "p95": 0.3329208791255951 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.071648359298706, "group": 8, "latest_source_mass": 0.11520501971244812, "layer": 32, "max_source_mass": 0.20955611765384674, "mixer_index": 62, "normalized_entropy": 0.9428478002054053, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.10569609701633453, "median": 0.08689188957214355, "p05": 0.03173588588833809, "p95": 0.24571751058101654 }, { "label": "completed_group_1", "mean": 0.09922641515731812, "median": 0.09795533865690231, "p05": 0.06407676637172699, "p95": 0.14120742678642273 }, { "label": "completed_group_2", "mean": 0.08180510997772217, "median": 0.080217644572258, "p05": 0.054607540369033813, "p95": 0.11442367732524872 }, { "label": "completed_group_3", "mean": 0.10646799206733704, "median": 0.10510502755641937, "p05": 0.07415887713432312, "p95": 0.14216329157352448 }, { "label": "completed_group_4", "mean": 0.0826798751950264, "median": 0.08022409677505493, "p05": 0.05491747707128525, "p95": 0.11860857903957367 }, { "label": "completed_group_5", "mean": 0.06615597754716873, "median": 0.06485182791948318, "p05": 0.04477996751666069, "p95": 0.09238749742507935 }, { "label": "completed_group_6", "mean": 0.20955611765384674, "median": 0.206287682056427, "p05": 0.129981130361557, "p95": 0.2996031939983368 }, { "label": "completed_group_7", "mean": 0.13320739567279816, "median": 0.12465103715658188, "p05": 0.06303662061691284, "p95": 0.23065781593322754 }, { "label": "current_group_partial", "mean": 0.11520501971244812, "median": 0.10386130958795547, "p05": 0.03912060335278511, "p95": 0.22735844552516937 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 1.9781701564788818, "group": 8, "latest_source_mass": 0.16112206876277924, "layer": 32, "max_source_mass": 0.22251944243907928, "mixer_index": 63, "normalized_entropy": 0.9003040366848136, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.09446848928928375, "median": 0.074150450527668, "p05": 0.042592622339725494, "p95": 0.19443637132644653 }, { "label": "completed_group_1", "mean": 0.07056106626987457, "median": 0.06868506222963333, "p05": 0.04818308353424072, "p95": 0.0982733890414238 }, { "label": "completed_group_2", "mean": 0.08033448457717896, "median": 0.08047192543745041, "p05": 0.048296794295310974, "p95": 0.11253741383552551 }, { "label": "completed_group_3", "mean": 0.0658978745341301, "median": 0.06256154924631119, "p05": 0.04223886504769325, "p95": 0.0967884361743927 }, { "label": "completed_group_4", "mean": 0.058234501630067825, "median": 0.0563921295106411, "p05": 0.03529864549636841, "p95": 0.08846980333328247 }, { "label": "completed_group_5", "mean": 0.04316519945859909, "median": 0.042268507182598114, "p05": 0.029443470761179924, "p95": 0.05996086448431015 }, { "label": "completed_group_6", "mean": 0.2036968618631363, "median": 0.19645483791828156, "p05": 0.09398645162582397, "p95": 0.3395216464996338 }, { "label": "completed_group_7", "mean": 0.22251944243907928, "median": 0.2130982130765915, "p05": 0.11867167800664902, "p95": 0.35465875267982483 }, { "label": "current_group_partial", "mean": 0.16112206876277924, "median": 0.14023703336715698, "p05": 0.06008770316839218, "p95": 0.3247341811656952 } ], "sources": 9 }, { "branch": "output", "entropy_mean": 1.6036605834960938, "group": null, "latest_source_mass": 0.31639498472213745, "layer": null, "max_source_mass": 0.3412485122680664, "mixer_index": null, "normalized_entropy": 0.7298573846467136, "offset": null, "source_summaries": [ { "label": "embedding", "mean": 0.09759747982025146, "median": 0.09915056824684143, "p05": 0.025461740791797638, "p95": 0.16398483514785767 }, { "label": "completed_group_1", "mean": 0.04215940088033676, "median": 0.04050712287425995, "p05": 0.025720715522766113, "p95": 0.06398092210292816 }, { "label": "completed_group_2", "mean": 0.041145481169223785, "median": 0.03881259262561798, "p05": 0.023510465398430824, "p95": 0.0662468671798706 }, { "label": "completed_group_3", "mean": 0.029554028064012527, "median": 0.027420252561569214, "p05": 0.01627083122730255, "p95": 0.04924973472952843 }, { "label": "completed_group_4", "mean": 0.0289996936917305, "median": 0.027226522564888, "p05": 0.01560397818684578, "p95": 0.048351090401411057 }, { "label": "completed_group_5", "mean": 0.03647579252719879, "median": 0.0347352996468544, "p05": 0.020744025707244873, "p95": 0.058257561177015305 }, { "label": "completed_group_6", "mean": 0.06642461568117142, "median": 0.057809896767139435, "p05": 0.022837381809949875, "p95": 0.14078694581985474 }, { "label": "completed_group_7", "mean": 0.3412485122680664, "median": 0.3336990475654602, "p05": 0.0819513276219368, "p95": 0.6423859596252441 }, { "label": "completed_group_8", "mean": 0.31639498472213745, "median": 0.30713334679603577, "p05": 0.05841828137636185, "p95": 0.6156357526779175 } ], "sources": 9 } ], "mode": "uniform_group_6_mlp_only", "optimizer_state_after": "ed03e6fbd4a12d8b063dcb22e0437754285f54d585374cd52fbd534f05d24637", "optimizer_state_before": "ed03e6fbd4a12d8b063dcb22e0437754285f54d585374cd52fbd534f05d24637", "optimizer_state_unchanged": true, "positions": { "attention_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "5e6ecbcd78543fc42d4098a51169dca446d404ce384015643e84d2c78433c949", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.0933690071105957, 0.24537226557731628, 0.07698935270309448, 0.0523640513420105, 0.09380833059549332, 0.03647654876112938, 0.039120711386203766, 0.06371714919805527, 0.05156813934445381, 0.03728010877966881, 0.03983496502041817, 0.040284499526023865, 0.046871740370988846, 0.04314620420336723, 0.04033593460917473, 0.04471700266003609, 0.08397205173969269, 0.0528092160820961, 0.04443390294909477, 0.07560808956623077, 0.03708760067820549, 0.03599191829562187, 0.033304885029792786, 0.04061548039317131, 0.025487789884209633, 0.021565141156315804, 0.02342291921377182, 0.017244853079319, 0.024166326969861984, 0.016819356009364128, 0.021590016782283783, 0.026201345026493073 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.7871444015327143e-05, "first_to_last_ratio": 0.7430739064285198, "imbalance_abs_log_ratio": 0.29695976891711484, "last_quartile_mean": 2.4050695174082648e-05, "mean": 1.940541733347345e-05, "normalized": [ 0.8528222225200949, 0.9904421919227359, 0.8889636070276252, 0.8260297081777587, 1.0592945922387769, 0.9917121294878884, 0.9337264690589573, 0.8246193538372446, 0.793822899201471, 0.7137497175026324, 0.6740780884121385, 0.6480585224874467, 0.7765430121048071, 0.7288936854726098, 0.6894394305490474, 0.6462023120775018, 0.7258916910482244, 0.6844708982483616, 0.6814006638957631, 0.6591586185576422, 1.6946272586080497, 1.5923311068806938, 1.5334129590810706, 1.4752651350864683, 2.3955034386643494, 2.1002622644164113, 1.5825019326053846, 1.0217356357060174, 1.0788741086745228, 0.7698719498369038, 0.509157554652168, 0.4571368419592321 ], "peak_layer": 25, "peak_normalized": 2.3955034386643494, "population_cv": 0.4552268456364437, "rest_mean": 1.675252436446802e-05, "spike_contrast": 2.013489945291234, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.373103936610278e-05, "top_five_layers": [ 25, 26, 21, 22, 27 ], "top_five_spike_overlap": 3 }, "values": [ 1.6549371139262803e-05, 1.9219944078940898e-05, 1.725070978864096e-05, 1.6029451217036694e-05, 2.055605364148505e-05, 1.9244587747380137e-05, 1.8119351807399653e-05, 1.6002082702470943e-05, 1.5404464647872373e-05, 1.3850611139787361e-05, 1.308076662098756e-05, 1.2575846085383091e-05, 1.5069141227286309e-05, 1.4144486158329528e-05, 1.337885987595655e-05, 1.2539825547719374e-05, 1.4086231203691568e-05, 1.32824434331269e-05, 1.322286425420316e-05, 1.2791248082066886e-05, 3.288494917796925e-05, 3.089984966209158e-05, 2.9756518415524624e-05, 2.8628135623876005e-05, 4.648574395105243e-05, 4.075646575074643e-05, 3.0709110433235765e-05, 1.9827206415357068e-05, 2.0936002329108305e-05, 1.4939686479920056e-05, 9.880414836516138e-06, 8.870931196724996e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 7.118063331290614e-05, "first_to_last_ratio": 0.7414541637100228, "imbalance_abs_log_ratio": 0.2991419349452631, "last_quartile_mean": 9.600139401300112e-05, "mean": 7.746012204279396e-05, "normalized": [ 0.8651509514970841, 1.0009621316887811, 0.8960527758772069, 0.8272828736395583, 1.0486092464922974, 0.9743231137662114, 0.9199918222512351, 0.8190880880203735, 0.7916641771719893, 0.7146356627542306, 0.6759717958986794, 0.6485427188091607, 0.7851038963161001, 0.7312400556921381, 0.6911330228585814, 0.6472807469974904, 0.7330555667584548, 0.68708616528045, 0.6823807528551565, 0.6566123404954054, 1.6860095573200757, 1.598354601512785, 1.532138419233743, 1.4724064331684281, 2.3914280353346062, 2.095428531328979, 1.577460819458396, 1.0184383982668763, 1.079420802550052, 0.7734487623850339, 0.5152991604613053, 0.46399857385913484 ], "peak_layer": 25, "peak_normalized": 2.3914280353346062, "population_cv": 0.4529486442223231, "rest_mean": 6.690162734701987e-05, "spike_contrast": 2.010055640387411, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013447599339997396, "top_five_layers": [ 25, 26, 21, 22, 27 ], "top_five_spike_overlap": 3 }, "values": [ 6.701469828840345e-05, 7.753464888082817e-05, 6.940835737623274e-05, 6.408143235603347e-05, 8.122540020849556e-05, 7.547118730144575e-05, 7.126267882995307e-05, 6.344666326185688e-05, 6.132240378065035e-05, 5.535576565307565e-05, 5.236085780779831e-05, 5.0236198148922995e-05, 6.081424362491816e-05, 5.664194395649247e-05, 5.3535248298430815e-05, 5.013844565837644e-05, 5.6782573665259406e-05, 5.322177821653895e-05, 5.2857296395814046e-05, 5.0861272029578686e-05, 0.00013059850607533008, 0.00012380874250084162, 0.00011867962894029915, 0.00011405278200982139, 0.00018524030747357756, 0.00016231214976869524, 0.0001221903075929731, 7.888836262281984e-05, 8.361206710105762e-05, 5.991143552819267e-05, 3.991513585788198e-05, 3.594138615881093e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.0631236988119781, "first_to_last_ratio": 0.7414541932050817, "imbalance_abs_log_ratio": 0.2991418951652474, "last_quartile_mean": 0.08513499470427632, "mean": 0.06869241083040833, "normalized": [ 0.8651510340629276, 1.0009621408154104, 0.8960527212303434, 0.8272828267127862, 1.048609258061384, 0.974323209748562, 0.9199918471122174, 0.8190881264681094, 0.7916641948980269, 0.7146356612347005, 0.6759717697820754, 0.6485427404539753, 0.7851038684126569, 0.7312400352642834, 0.691133044440946, 0.6472807741872908, 0.733055542127618, 0.6870861838155687, 0.6823807362394988, 0.6566123287013491, 1.6860096010442378, 1.5983546280989276, 1.5321384391908206, 1.4724063815551618, 2.3914281361629333, 2.095428443818053, 1.5774607618549192, 1.0184383399289305, 1.0794207582970639, 0.7734487124090256, 0.515299168335352, 0.4639985855348445 ], "peak_layer": 25, "peak_normalized": 2.3914281361629333, "population_cv": 0.4529486466170061, "rest_mean": 0.0593290317941595, "spike_contrast": 2.0100556847093496, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.11925465762615203, "top_five_layers": [ 25, 26, 21, 22, 27 ], "top_five_spike_overlap": 3 }, "values": [ 0.05942931026220322, 0.06875850260257721, 0.0615520216524601, 0.056828051805496216, 0.07203149795532227, 0.06692861020565033, 0.06319645792245865, 0.05626513808965683, 0.054381322115659714, 0.04909004643559456, 0.046434130519628525, 0.04454996436834335, 0.053930677473545074, 0.050230640918016434, 0.04747559502720833, 0.044463276863098145, 0.05035535246133804, 0.0471976064145565, 0.04687437787652016, 0.04510428383946419, 0.11581606417894363, 0.10979483276605606, 0.10524628311395645, 0.10114314407110214, 0.1642729640007019, 0.1439400315284729, 0.10835958272218704, 0.06995898485183716, 0.0741480141878128, 0.05313005670905113, 0.03539714217185974, 0.0318731814622879 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 2.8004650971524825e-06, "first_to_last_ratio": 0.4310298271707096, "imbalance_abs_log_ratio": 0.8415779867005032, "last_quartile_mean": 6.49714920086808e-06, "mean": 4.228376049297822e-06, "normalized": [ 0.7549623656313754, 0.6713253858936958, 0.6327881150618933, 0.6240061846000786, 0.6846720230741171, 0.6541307323530884, 0.6366162353061869, 0.639920980155935, 0.5854762807084849, 0.5554311560991867, 0.5559610918346484, 0.5665898120375786, 0.6177325090728767, 0.6174014807230541, 0.628670963404296, 0.6213050448878809, 0.6162839642893286, 0.6395459114847178, 0.6569181781896307, 0.6901984112430531, 1.6002608407680152, 1.7304442459033893, 1.8467568318645033, 1.8801293936503243, 2.939844491857359, 2.632194185755028, 2.0221642159701534, 1.305227038166136, 1.2879360228966652, 0.9294683481405218, 0.6267978785262305, 0.5488396804505664 ], "peak_layer": 25, "peak_normalized": 2.939844491857359, "population_cv": 0.6404446436976945, "rest_mean": 3.4457450173822173e-06, "spike_contrast": 2.4536300797048405, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 8.454583621642087e-06, "top_five_layers": [ 25, 26, 27, 24, 23 ], "top_five_spike_overlap": 3 }, "values": [ 3.192264784956933e-06, 2.8386161829985213e-06, 2.6756661100080237e-06, 2.6385328055766877e-06, 2.8950507839908823e-06, 2.7659107217914425e-06, 2.6918528419628274e-06, 2.705826545934542e-06, 2.475613882779726e-06, 2.3485717974836007e-06, 2.3508125650550937e-06, 2.395754790995852e-06, 2.6120053462364012e-06, 2.6106056338903727e-06, 2.6582572445477126e-06, 2.6271113711118232e-06, 2.6058803541673115e-06, 2.7042406145483255e-06, 2.7776970910053933e-06, 2.918418431363534e-06, 6.766504611732671e-06, 7.316969004023122e-06, 7.80878235673299e-06, 7.949894097691868e-06, 1.2430768038029782e-05, 1.1129906852147542e-05, 8.550470738555305e-06, 5.518990747077623e-06, 5.44587783224415e-06, 3.930141701857792e-06, 2.6503371373109985e-06, 2.3207005597214447e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 5.2933271490474e-05, "first_to_last_ratio": 0.8910427239585592, "imbalance_abs_log_ratio": 0.11536290209014609, "last_quartile_mean": 5.940598589404544e-05, "mean": 5.372000583747649e-05, "normalized": [ 0.9420203782851446, 1.065992822363657, 0.9637067116209882, 0.8975585528867911, 1.1255456215013835, 1.0475662450004608, 0.9767355773440091, 0.8637133670195876, 0.8461142739864999, 0.7632528144311652, 0.7157547804677827, 0.6814463821594676, 0.8446324682227626, 0.7783613279398726, 0.7274401946693456, 0.6746038404855031, 0.7942303495938553, 0.7353930917796355, 0.7290118663555032, 0.7029917878505813, 1.7662218595201464, 1.6279985898812712, 1.5042083156078159, 1.3787408601477276, 2.2602835719986163, 1.8755133135023399, 1.4006178961493854, 0.9397316076897866, 0.8802451075257222, 0.6257902561009169, 0.4518812760737081, 0.41269489183856706 ], "peak_layer": 25, "peak_normalized": 2.2602835719986163, "population_cv": 0.4162315989727347, "rest_mean": 4.6681783378180286e-05, "spike_contrast": 1.9649293681558488, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.172640711767599e-05, "top_five_layers": [ 25, 26, 21, 22, 23 ], "top_five_spike_overlap": 4 }, "values": [ 5.0605340220499784e-05, 5.72651406400837e-05, 5.177033017389476e-05, 4.821685070055537e-05, 6.046431735740043e-05, 5.627526479656808e-05, 5.247024091659114e-05, 4.639868711819872e-05, 4.5453263737726957e-05, 4.1001945646712556e-05, 3.845035098493099e-05, 3.660730362753384e-05, 4.537366112344898e-05, 4.181357508059591e-05, 3.907809150405228e-05, 3.623972224886529e-05, 4.2666059016482905e-05, 3.9505321183241904e-05, 3.9162521716207266e-05, 3.776472294703126e-05, 9.488144860370085e-05, 8.745609375182539e-05, 8.080607949523255e-05, 7.406596705550328e-05, 0.00012142244668211788, 0.00010075258614961058, 7.524120155721903e-05, 5.0482387450756505e-05, 4.7286772314691916e-05, 3.3617456210777164e-05, 2.4275064788525924e-05, 2.216997199866455e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 3.816239222942386e-05, "first_to_last_ratio": 1.0982104653328248, "imbalance_abs_log_ratio": 0.09368200534994196, "last_quartile_mean": 3.474961624760908e-05, "mean": 3.635289348835613e-05, "normalized": [ 1.0100679155909518, 1.1276705560073144, 1.0341062865012667, 0.9738227205093075, 1.2081657639134786, 1.1263936119975244, 1.017692752424331, 0.9002877581223364, 0.906435102686081, 0.8204731533674717, 0.7644113584154034, 0.724335581489344, 0.9145695162355928, 0.8387322017826943, 0.7801480421153628, 0.7183641669858825, 0.8749039925985721, 0.8055826452219621, 0.7932146519026885, 0.7731819923052161, 1.890055266177086, 1.6611840679584398, 1.4700335556114335, 1.2189925678736009, 2.0493580343576667, 1.5904309621219501, 1.1902855455569599, 0.8755158449242662, 0.6989731317237368, 0.4945346959783037, 0.38555938350958924, 0.3625171740341845 ], "peak_layer": 25, "peak_normalized": 2.0493580343576667, "population_cv": 0.3831879266904683, "rest_mean": 3.192373302833099e-05, "spike_contrast": 1.8879483774345691, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.027035997249186e-05, "top_five_layers": [ 25, 21, 22, 26, 23 ], "top_five_spike_overlap": 4 }, "values": [ 3.671889135148376e-05, 4.099408761248924e-05, 3.7592755688820034e-05, 3.540127363521606e-05, 4.392032133182511e-05, 4.094766700291075e-05, 3.6996076232753694e-05, 3.2728064979892224e-05, 3.2951538742054254e-05, 2.9826573154423386e-05, 2.7788564693764783e-05, 2.6331694243708625e-05, 3.32472482114099e-05, 3.0490342396660708e-05, 2.836063868016936e-05, 2.6114616048289463e-05, 3.180529165547341e-05, 2.9285260097822174e-05, 2.883564775402192e-05, 2.8107402613386512e-05, 6.87089777784422e-05, 6.0388847487047315e-05, 5.343997327145189e-05, 4.4313906983006746e-05, 7.450009434251115e-05, 5.781676736660302e-05, 4.327032365836203e-05, 3.182753425789997e-05, 2.5409695808775723e-05, 1.7977767129195854e-05, 1.4016199202160351e-05, 1.3178548215364572e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00014839320101600606, "first_to_last_ratio": 0.7316308375057152, "imbalance_abs_log_ratio": 0.3124792125804878, "last_quartile_mean": 0.00020282524110371014, "mean": 0.00016261010296148015, "normalized": [ 0.8552830184806287, 0.9767466441492901, 0.8836897064219967, 0.8123623681209048, 1.04432087755931, 0.9707919157924664, 0.9325102280056078, 0.8248601295522114, 0.7951368704203655, 0.714969927099356, 0.6759729488336033, 0.6462738071525346, 0.7765153347527617, 0.7268833213559875, 0.6836287405043341, 0.634168412930625, 0.7327540189659476, 0.6689370517140172, 0.669926896326434, 0.6397518041371708, 1.6758122087088618, 1.5988232205344164, 1.556805520372621, 1.5245933761164159, 2.469780881717557, 2.118963162825585, 1.6204612720862392, 1.0552031724918056, 1.0636927836342112, 0.7271989959668094, 0.4831976455461727, 0.43998373772375193 ], "peak_layer": 25, "peak_normalized": 2.469780881717557, "population_cv": 0.47103212933078636, "rest_mean": 0.00013956876204314608, "spike_contrast": 2.0565729731969045, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0002870333439204842, "top_five_layers": [ 25, 26, 21, 27, 22 ], "top_five_spike_overlap": 3 }, "values": [ 0.00013907765969634056, 0.0001588288723723963, 0.00014369687414728105, 0.00013209832832217216, 0.0001698171254247427, 0.00015786057338118553, 0.00015163558418862522, 0.00013413059059530497, 0.00012929728836752474, 0.00011626133345998824, 0.00010992003080900759, 0.00010509065032238141, 0.0001262692385353148, 0.0001181985717266798, 0.00011116493988083676, 0.00010312219092156738, 0.00011915320646949112, 0.00010877592285396531, 0.00010893688158830628, 0.00010403010674053803, 0.0002725039958022535, 0.00025998480850830674, 0.00025315230595879257, 0.000247914285864681, 0.0004016113234683871, 0.000344564818078652, 0.0002635033742990345, 0.00017158669652417302, 0.0001729671930661425, 0.00011824990360764787, 7.857281889300793e-05, 7.154580089263618e-05 ] } } }, "mlp_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "c0b8018c349b1bd09e5d17858eb312a2323da0e92de9651e2bb08ae4adf115c7", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.13583384454250336, 0.0415157787501812, 0.018140379339456558, 0.039518751204013824, 0.022877270355820656, 0.015532293356955051, 0.02314678393304348, 0.05432315170764923, 0.03743875026702881, 0.04006272554397583, 0.045583780854940414, 0.05995568633079529, 0.038651175796985626, 0.04559725895524025, 0.05632574483752251, 0.07952121645212173, 0.042337141931056976, 0.05044656619429588, 0.04418695345520973, 0.06310147047042847, 0.029010163620114326, 0.04586305841803551, 0.06033914536237717, 0.07656749337911606, 0.045884523540735245, 0.051604680716991425, 0.06502768397331238, 0.06792232394218445, 0.060306549072265625, 0.08025261014699936, 0.09663844108581543, 0.12230555713176727 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.7712873386699357e-05, "first_to_last_ratio": 0.9452196471532902, "imbalance_abs_log_ratio": 0.05633794764621688, "last_quartile_mean": 1.8739425740932347e-05, "mean": 1.785050199032412e-05, "normalized": [ 0.9253507487414744, 1.075518418418416, 0.9670536529973797, 0.8963702221058623, 1.1468724617818529, 1.0665333729695432, 0.9882456592783722, 0.8723749109923966, 0.8003566239884481, 0.747382060005652, 0.7114449352492375, 0.6868613479162928, 0.820669680584508, 0.7666628564071297, 0.7137970720647795, 0.6764342971353837, 0.7833155163151755, 0.7287358030149997, 0.723203785103502, 0.7295485678039421, 1.8338227695352503, 1.7117668159885895, 1.6511497052670414, 1.5781427208753402, 2.3061795319056673, 1.7327146667487312, 1.1180986928890646, 0.8849233430892791, 0.836865618854635, 0.5568714099056575, 0.49886857710039795, 0.4638641549659982 ], "peak_layer": 25, "peak_normalized": 2.3061795319056673, "population_cv": 0.42887425312420646, "rest_mean": 1.515239098268837e-05, "spike_contrast": 2.1396162142725466, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.242030143155716e-05, "top_five_layers": [ 25, 21, 26, 22, 23 ], "top_five_spike_overlap": 4 }, "values": [ 1.65179753821576e-05, 1.919854366860818e-05, 1.7262393157579936e-05, 1.6000658433767967e-05, 2.0472249161684886e-05, 1.903815609693993e-05, 1.7640681107877754e-05, 1.5572330084978603e-05, 1.4286767509474885e-05, 1.334114494966343e-05, 1.2699649232672527e-05, 1.2260819858056493e-05, 1.4649365766672418e-05, 1.3685316844203044e-05, 1.2741636055579875e-05, 1.2074691767338663e-05, 1.3982575183035806e-05, 1.3008299902139697e-05, 1.29095506054e-05, 1.3022808161622379e-05, 3.2734656997490674e-05, 3.05558969557751e-05, 2.9473851100192405e-05, 2.8170639780000784e-05, 4.116646232432686e-05, 3.092982660746202e-05, 1.9958622942795046e-05, 1.579632589709945e-05, 1.4938471394998487e-05, 9.940434210875537e-06, 8.905054528440814e-06, 8.280208021460567e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 7.054649267956847e-05, "first_to_last_ratio": 0.9422119032705132, "imbalance_abs_log_ratio": 0.0595250793099939, "last_quartile_mean": 7.487327684430056e-05, "mean": 7.126561581571877e-05, "normalized": [ 0.9379867260928122, 1.0867975337598494, 0.9748719850049338, 0.8973991480757045, 1.13511543726997, 1.0472042718711223, 0.9740366328576487, 0.8658623073483388, 0.7997865344471587, 0.7493453336250012, 0.7134775156593193, 0.6873380993536559, 0.8313371071305022, 0.7700028918465563, 0.7167677739886463, 0.678061218699671, 0.7908636217587858, 0.7311166980702738, 0.7234345620435917, 0.7232398132998926, 1.8221876777881907, 1.7164038356786036, 1.647204987183513, 1.5751763347937393, 2.3017788437669457, 1.7269305795673797, 1.1110598069978055, 0.8827581264026101, 0.840924462714809, 0.5633854501369467, 0.5064065620653, 0.4717381207007217 ], "peak_layer": 25, "peak_normalized": 2.3017788437669457, "population_cv": 0.4259359014777558, "rest_mean": 6.054211580367e-05, "spike_contrast": 2.1335976479525653, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012917251588078215, "top_five_layers": [ 25, 21, 26, 22, 23 ], "top_five_spike_overlap": 4 }, "values": [ 6.684620166197419e-05, 7.745129551040009e-05, 6.947485235286877e-05, 6.395370292011648e-05, 8.08947006589733e-05, 7.462965731974691e-05, 6.941532046766952e-05, 6.170621054479852e-05, 5.6997279898496345e-05, 5.340255665942095e-05, 5.084641452413052e-05, 4.898357292404398e-05, 5.924575089011341e-05, 5.487473026732914e-05, 5.108089681016281e-05, 4.832245031138882e-05, 5.6361383030889556e-05, 5.2103481721132994e-05, 5.1556009566411376e-05, 5.154213067726232e-05, 0.00012985932698938996, 0.00012232057633809745, 0.00011738907778635621, 0.00011225591151742265, 0.00016403768677264452, 0.00012307077122386545, 7.918036135379225e-05, 6.291030149441212e-05, 5.992899968987331e-05, 4.015001104562543e-05, 3.608937549870461e-05, 3.36187076754868e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.06256133550778031, "first_to_last_ratio": 0.9422119063248422, "imbalance_abs_log_ratio": 0.05952507606833563, "last_quartile_mean": 0.06639837077818811, "mean": 0.06319906114367768, "normalized": [ 0.937986741337558, 1.0867974412744201, 0.9748719661802793, 0.8973991595387034, 1.1351154100023004, 1.047204326347193, 0.9740366517331313, 0.8658623403386309, 0.7997864977593775, 0.7493453767435976, 0.7134774895653551, 0.6873381185328077, 0.8313371101695717, 0.770002889878303, 0.7167677596327712, 0.6780611826552376, 0.7908636465536031, 0.7311167064424772, 0.7234345368035344, 0.7232398403375878, 1.8221877439862273, 1.7164038430620674, 1.647204958351786, 1.5751763435342587, 2.3017788388197298, 1.7269305380891986, 1.1110598157689295, 0.8827581385574063, 0.8409244522202193, 0.5633854357754055, 0.5064065689433755, 0.47173813106495593 ], "peak_layer": 25, "peak_normalized": 2.3017788388197298, "population_cv": 0.4259359013380368, "rest_mean": 0.05368935392686614, "spike_contrast": 2.1335976638962397, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.11455148011446, "top_five_layers": [ 25, 21, 26, 22, 23 ], "top_five_spike_overlap": 4 }, "values": [ 0.05927988141775131, 0.06868457794189453, 0.06161099299788475, 0.05671478435397148, 0.07173822820186615, 0.06618233025074005, 0.06155820190906525, 0.054721686989068985, 0.05054575577378273, 0.04735792428255081, 0.04509110748767853, 0.043439123779535294, 0.052539724856615067, 0.048663459718227386, 0.04529904946684837, 0.04285283014178276, 0.04998183995485306, 0.04620588943362236, 0.045720383524894714, 0.045708078891038895, 0.11516055464744568, 0.10847511142492294, 0.10410180687904358, 0.09954966604709625, 0.1454702615737915, 0.10914038866758347, 0.07021793723106384, 0.055789485573768616, 0.0531456358730793, 0.035605430603027344, 0.03200441971421242, 0.029813406988978386 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 2.7768887775891926e-06, "first_to_last_ratio": 0.5399692621850023, "imbalance_abs_log_ratio": 0.616243062923554, "last_quartile_mean": 5.142679356140434e-06, "mean": 3.843927927960067e-06, "normalized": [ 0.8226986918348728, 0.7374612447956529, 0.6983137318015623, 0.686878644467254, 0.756301790252643, 0.7120677310315813, 0.688169682679528, 0.6773818298416324, 0.6119386261012653, 0.5946615110020419, 0.5997068878623979, 0.6093634112378197, 0.6648009209545905, 0.6563460580429435, 0.6542077353372848, 0.64785458042162, 0.6725755428948662, 0.6822023131217141, 0.692090828244068, 0.7477479670675087, 1.7531073305905795, 1.8850714763755212, 2.019469820112974, 2.0266143614595933, 2.9220024204674364, 2.2546323180910592, 1.518053907773469, 1.1372024855664973, 1.0131254676633572, 0.6851695833902249, 0.61233369822716, 0.5604474012892807 ], "peak_layer": 25, "peak_normalized": 2.9220024204674364, "population_cv": 0.6006089418238398, "rest_mean": 3.0457768102846955e-06, "spike_contrast": 2.677131146272305, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 8.153943963407072e-06, "top_five_layers": [ 25, 26, 24, 23, 22 ], "top_five_spike_overlap": 4 }, "values": [ 3.16239447784028e-06, 2.8347478746582055e-06, 2.684267656150041e-06, 2.640312004587031e-06, 2.907169573518331e-06, 2.7371370379114524e-06, 2.645274662427255e-06, 2.6038069336209446e-06, 2.352247975068167e-06, 2.2858359898236813e-06, 2.3052300548442872e-06, 2.3423490347340703e-06, 2.5554468265909236e-06, 2.5229469429177698e-06, 2.514727384550497e-06, 2.4903063149395166e-06, 2.5853319129964802e-06, 2.6223365239275154e-06, 2.660347263372387e-06, 2.874289293686161e-06, 6.7388182287686504e-06, 7.246078894240782e-06, 7.762696441204753e-06, 7.790159543219488e-06, 1.1231966709601693e-05, 8.666644134791568e-06, 5.835289812239353e-06, 4.371324394014664e-06, 3.8943812796787824e-06, 2.6337424969824497e-06, 2.353766603846452e-06, 2.1543194179685088e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 5.252117625786923e-05, "first_to_last_ratio": 1.1102803930941318, "imbalance_abs_log_ratio": 0.10461258981573004, "last_quartile_mean": 4.730442560685333e-05, "mean": 5.009002063616208e-05, "normalized": [ 1.0080949148570253, 1.1419217848905152, 1.0344343353936138, 0.9611575268119311, 1.2029231617943856, 1.1121831905870352, 1.0229568038697068, 0.904614106529825, 0.8482368609748383, 0.7909016413592282, 0.7457024060979733, 0.7135814414278084, 0.8839804056446833, 0.8102519910956801, 0.7458758437065229, 0.6981703333213867, 0.8462569994970068, 0.7745627001501176, 0.7649143253143685, 0.7696378128812155, 1.8857975763088313, 1.7268627799140321, 1.5952974259002308, 1.456577843162574, 2.107397821000754, 1.54790596250813, 1.0247915529765659, 0.8316989168145444, 0.676980005203446, 0.492124311814465, 0.4471995771568589, 0.42700764103469996 ], "peak_layer": 25, "peak_normalized": 2.107397821000754, "population_cv": 0.39981055259953574, "rest_mean": 4.3092382704209604e-05, "spike_contrast": 2.0392760844046087, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 8.787726546870545e-05, "top_five_layers": [ 25, 21, 22, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 5.0495495088398457e-05, 5.7198885770048946e-05, 5.181483720662072e-05, 4.814440035261214e-05, 6.025444599799812e-05, 5.570927896769717e-05, 5.123992741573602e-05, 4.5312139263842255e-05, 4.2488201870583e-05, 3.96162795368582e-05, 3.73522489098832e-05, 3.5743309126701206e-05, 4.4278596760705113e-05, 4.058553895447403e-05, 3.7360936403274536e-05, 3.4971366403624415e-05, 4.238903056830168e-05, 3.8797861634520814e-05, 3.8314574339892715e-05, 3.855117392959073e-05, 9.445963951293379e-05, 8.649859228171408e-05, 7.990848098415881e-05, 7.296001422218978e-05, 0.00010555960034253076, 7.753464160487056e-05, 5.133183003636077e-05, 4.165981590631418e-05, 3.390994243090972e-05, 2.4650516934343614e-05, 2.2400236048270017e-05, 2.138882155122701e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 3.791993913182523e-05, "first_to_last_ratio": 1.3353078460651115, "imbalance_abs_log_ratio": 0.2891618615724262, "last_quartile_mean": 2.8397900337040483e-05, "mean": 3.4411754228358404e-05, "normalized": [ 1.0637575904901695, 1.1893182322665659, 1.0929436668106773, 1.0287748315987029, 1.2701021943757262, 1.1768025739191794, 1.05468055075822, 0.9391986369441508, 0.9024922342629136, 0.8434659977082996, 0.7878104552637905, 0.7485220227678168, 0.9461119243794104, 0.863069485682856, 0.7893906382695377, 0.7346328622052256, 0.9187021399477474, 0.8378587637187482, 0.8203924925531959, 0.8399030534613141, 1.994363063906111, 1.7401733899365923, 1.5375765272401527, 1.2780496257972183, 1.8396423527015207, 1.312671988363898, 0.948992769158069, 0.809919115285961, 0.524065785626751, 0.417236660898879, 0.38395732702419516, 0.36542104667640435 ], "peak_layer": 21, "peak_normalized": 1.994363063906111, "population_cv": 0.3800700971389524, "rest_mean": 3.009141588942536e-05, "spike_contrast": 1.918872195006558, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.774158125859685e-05, "top_five_layers": [ 21, 25, 22, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 3.660576476249844e-05, 4.092652670806274e-05, 3.761010884772986e-05, 3.540194666129537e-05, 4.3706444557756186e-05, 4.049584094900638e-05, 3.6293407902121544e-05, 3.231947266613133e-05, 3.105634095845744e-05, 2.9025144613115117e-05, 2.7109939765068702e-05, 2.5757955881999806e-05, 3.255737101426348e-05, 2.9699735023314133e-05, 2.7164316634298302e-05, 2.528000550228171e-05, 3.161415224894881e-05, 2.883218985516578e-05, 2.823114482453093e-05, 2.8902537451358512e-05, 6.862953159725294e-05, 5.9882419009227306e-05, 5.291070556268096e-05, 4.3979929614579305e-05, 6.330532050924376e-05, 4.5171345846029e-05, 3.265650593675673e-05, 2.7870737540069968e-05, 1.8034023014479317e-05, 1.435784542991314e-05, 1.3212645171734039e-05, 1.2574779248097911e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00014698960330861155, "first_to_last_ratio": 0.9342716987251417, "imbalance_abs_log_ratio": 0.06798798506325479, "last_quartile_mean": 0.00015733068175904918, "mean": 0.00014926503831702576, "normalized": [ 0.923079030839124, 1.0632412407145433, 0.9637007651329002, 0.883836006439372, 1.1353318187349588, 1.0485717557432739, 0.9951638569231974, 0.8651214478630361, 0.8092736556390749, 0.7468754803165791, 0.7147314665028897, 0.6894687183630649, 0.8220373973626863, 0.7589912517877063, 0.7049476665114015, 0.6634194660463175, 0.7942705096147508, 0.7203879109524882, 0.7123737088001584, 0.7065063437696151, 1.8198299866845873, 1.7148056734888262, 1.6770084324102719, 1.634740669418242, 2.3107097881677157, 1.7766651166041112, 1.1538175672334219, 0.9305534283893316, 0.7926512420440541, 0.5314353885089727, 0.47945177636664654, 0.4570014326266808 ], "peak_layer": 25, "peak_normalized": 2.3107097881677157, "population_cv": 0.43816618240664734, "rest_mean": 0.0001262832280459676, "spike_contrast": 2.1647119574836435, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0002733668137807399, "top_five_layers": [ 25, 21, 26, 22, 23 ], "top_five_spike_overlap": 4 }, "values": [ 0.00013778342690784484, 0.00015870474453549832, 0.00014384683163370937, 0.00013192581536713988, 0.00016946534742601216, 0.00015651510329917073, 0.0001485431712353602, 0.0001291323860641569, 0.00012079626321792603, 0.00011148239718750119, 0.00010668441973393783, 0.00010291357466485351, 0.0001227014436153695, 0.00011329085828037933, 0.00010522404045332223, 9.902533201966435e-05, 0.00011855681805172935, 0.00010752872913144529, 0.00010633248894009739, 0.00010545669647399336, 0.0002716369926929474, 0.0002559605345595628, 0.0002503187279216945, 0.00024400962865911424, 0.00034490818507038057, 0.0002651939867064357, 0.00017222462338395417, 0.00013889909314457327, 0.00011831511801574379, 7.932472362881526e-05, 7.156538777053356e-05, 6.821433635195717e-05 ] } } }, "post_attention_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "90a633f12d984935028fe5d7212be48e7da7b4a0806751939a551938d16dc540", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.0933690071105957, 0.35791492462158203, 0.3940892517566681, 0.4057151973247528, 0.09380833059549332, 0.11286329478025436, 0.13231655955314636, 0.1621972620487213, 0.05156813934445381, 0.08533531427383423, 0.12003090232610703, 0.16063299775123596, 0.046871740370988846, 0.079384446144104, 0.11459959298372269, 0.15625107288360596, 0.08397205173969269, 0.11654024571180344, 0.15294702351093292, 0.21572710573673248, 0.03708760067820549, 0.06553087383508682, 0.09390194714069366, 0.12852425873279572, 0.025487789884209633, 0.06059965491294861, 0.08860157430171967, 0.12100546807050705, 0.024166326969861984, 0.06877554953098297, 0.12682415544986725, 0.2025238573551178 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.787151654752961e-05, "first_to_last_ratio": 0.7430707227226617, "imbalance_abs_log_ratio": 0.29696405343397225, "last_quartile_mean": 2.4050895831351227e-05, "mean": 1.9405512830417138e-05, "normalized": [ 0.8527913110002082, 0.9904387238690049, 0.8889577325558207, 0.8260531077439892, 1.0592985654172338, 0.9917423063023154, 0.9337078137192232, 0.8246143584257407, 0.7938156182148532, 0.713755719225385, 0.6740731776830099, 0.6480627384314089, 0.776538956300743, 0.7288885518569582, 0.6894409119884075, 0.6462047561777335, 0.7258969300010353, 0.6844768097165681, 0.6814020911591906, 0.6591517190635972, 1.6946267929355336, 1.592341268076286, 1.5333897591025218, 1.4752526259237244, 2.395488463079873, 2.100311919614885, 1.5824933950307984, 1.0217488860800465, 1.078852489388298, 0.7698869552098468, 0.509167515873087, 0.45712803083267256 ], "peak_layer": 25, "peak_normalized": 2.395488463079873, "population_cv": 0.45522779950739584, "rest_mean": 1.6752636266001327e-05, "spike_contrast": 2.0134769085100985, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.373104627826251e-05, "top_five_layers": [ 25, 26, 21, 22, 27 ], "top_five_spike_overlap": 3 }, "values": [ 1.6548852727282792e-05, 1.921997136378195e-05, 1.7250680684810504e-05, 1.6029984180931933e-05, 2.0556231902446598e-05, 1.9245268049417064e-05, 1.811907895898912e-05, 1.6002064512576908e-05, 1.5404399164253846e-05, 1.3850795767211821e-05, 1.30807356981677e-05, 1.2575989785545971e-05, 1.50691366798128e-05, 1.414445614500437e-05, 1.3378954463405535e-05, 1.2539934687083587e-05, 1.4086402188695502e-05, 1.3282623513077851e-05, 1.322295702266274e-05, 1.2791177141480148e-05, 3.2885101973079145e-05, 3.0900198908057064e-05, 2.9756214644294232e-05, 2.8628033760469407e-05, 4.648568210541271e-05, 4.07576299039647e-05, 3.0709095881320536e-05, 1.982756111829076e-05, 2.0935685824952088e-05, 1.4940051187295467e-05, 9.88065676210681e-06, 8.870803867466748e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 7.118078883650014e-05, "first_to_last_ratio": 0.7414535710513813, "imbalance_abs_log_ratio": 0.29914273426490734, "last_quartile_mean": 9.600168050383218e-05, "mean": 7.746031667466013e-05, "normalized": [ 0.8651378816172938, 1.0009777453700024, 0.8960513697790896, 0.8272911274107208, 1.0485994729000152, 0.9743275226118578, 0.919982653625782, 0.8190908204284537, 0.7916631272989247, 0.7146362153993723, 0.6759631464825095, 0.6485478992662592, 0.7850933288875983, 0.7312355412849723, 0.6911368282263086, 0.6472758799634579, 0.7330540066292776, 0.6870915776507333, 0.6823867876028037, 0.6566142130765098, 1.6860192227923136, 1.5983806434370142, 1.5321192586607606, 1.4723959704412555, 2.391407748899872, 2.0954557664896343, 1.577449717039006, 1.018449459327879, 1.079403155230695, 0.7734578559089683, 0.5153068361375799, 0.4639972201230792 ], "peak_layer": 25, "peak_normalized": 2.391407748899872, "population_cv": 0.4529484648515137, "rest_mean": 6.69018361939514e-05, "spike_contrast": 2.010051127455382, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001344761112704873, "top_five_layers": [ 25, 26, 21, 22, 27 ], "top_five_spike_overlap": 3 }, "values": [ 6.70138542773202e-05, 7.753605314064771e-05, 6.940842285985127e-05, 6.408223271137103e-05, 8.122484723571688e-05, 7.547171844635159e-05, 7.126214768504724e-05, 6.344703433569521e-05, 6.132247654022649e-05, 5.5355947552016005e-05, 5.236031938693486e-05, 5.023672565585002e-05, 6.0813577874796465e-05, 5.6641736591700464e-05, 5.353567757993005e-05, 5.013819463783875e-05, 5.678259549313225e-05, 5.322233118931763e-05, 5.2857896662317216e-05, 5.086154487798922e-05, 0.00013059958291705698, 0.00012381107080727816, 0.00011867844295920804, 0.00011405225814087316, 0.0001852392015280202, 0.00016231466725002974, 0.00012218975462019444, 7.88894176366739e-05, 8.361091022379696e-05, 5.9912290453212336e-05, 3.991583071183413e-05, 3.59413716068957e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.06312383618205786, "first_to_last_ratio": 0.7414535762279484, "imbalance_abs_log_ratio": 0.2991427272832604, "last_quartile_mean": 0.08513525081798434, "mean": 0.06869258382357657, "normalized": [ 0.865137846334376, 1.0009777332897074, 0.8960513323416298, 0.8272911557263163, 1.0485995672048236, 0.9743275891924833, 0.9199825886242237, 0.8190908360604096, 0.7916630688979818, 0.7146361934667643, 0.6759631800576371, 0.6485479403328315, 0.7850933769135411, 0.7312355363979158, 0.6911368355098055, 0.6472758359852144, 0.7330539671810486, 0.6870915577845296, 0.6823867728344861, 0.6566142001467832, 1.686019238273426, 1.5983806470123938, 1.5321192874676839, 1.4723959488097038, 2.3914077965965506, 2.0954557055582517, 1.5774497391559377, 1.0184494414225371, 1.0794031805340594, 0.7734578820039498, 0.5153068187924214, 0.4639972000905758 ], "peak_layer": 25, "peak_normalized": 2.3914077965965506, "population_cv": 0.4529484701923164, "rest_mean": 0.059329217506779566, "spike_contrast": 2.010051150946886, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.11925476193428039, "top_five_layers": [ 25, 26, 21, 22, 27 ], "top_five_spike_overlap": 3 }, "values": [ 0.05942855402827263, 0.0687597468495369, 0.061552081257104874, 0.05682876706123352, 0.07203101366758347, 0.06692907959222794, 0.06319598108530045, 0.056265465915203094, 0.05438138172030449, 0.04909020662307739, 0.04643365740776062, 0.04455043375492096, 0.053930092602968216, 0.05023045837879181, 0.04747597500681877, 0.04446304962038994, 0.050355371087789536, 0.0471980944275856, 0.04687491059303284, 0.04510452598333359, 0.11581701785326004, 0.10979689657688141, 0.10524523258209229, 0.10114268213510513, 0.1642719805240631, 0.14394226670265198, 0.10835909843444824, 0.06995992362499237, 0.07414699345827103, 0.05313082039356232, 0.03539775684475899, 0.03187316656112671 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 2.8003725844882865e-06, "first_to_last_ratio": 0.43101667801433924, "imbalance_abs_log_ratio": 0.8416084935381094, "last_quartile_mean": 6.497132773120029e-06, "mean": 4.228344018031294e-06, "normalized": [ 0.7549446394195485, 0.6712955723041529, 0.6327934464023489, 0.6240143531936214, 0.6845982699352605, 0.6541306329076181, 0.6366139597815825, 0.6398962522620891, 0.5854979234893336, 0.5554041211792402, 0.5560120865584995, 0.5665670022229725, 0.6176883621108827, 0.6174393361360597, 0.6286877711262276, 0.6213255609688095, 0.6162655101795895, 0.6395309137881207, 0.6569538055926076, 0.6901619113645622, 1.60026726332299, 1.7304476753923834, 1.8467983538602002, 1.8801313759222678, 2.939904834039451, 2.632194121783472, 2.022104036344685, 1.305265318252875, 1.287886520877098, 0.929482057144306, 0.6268228994339659, 0.5488741127031785 ], "peak_layer": 25, "peak_normalized": 2.939904834039451, "population_cv": 0.6404496542420026, "rest_mean": 3.445701108999115e-06, "spike_contrast": 2.4536706636348105, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 8.45461572680506e-06, "top_five_layers": [ 25, 26, 27, 24, 23 ], "top_five_spike_overlap": 3 }, "values": [ 3.1921656500344398e-06, 2.8384686174831586e-06, 2.675668383744778e-06, 2.638547357491916e-06, 2.8947169994353317e-06, 2.765889348665951e-06, 2.691822828637669e-06, 2.7057014904130483e-06, 2.4756866423558677e-06, 2.348439693378168e-06, 2.3510103801527293e-06, 2.3956401946634287e-06, 2.6117988909390988e-06, 2.610745923448121e-06, 2.658308176251012e-06, 2.6271782189724036e-06, 2.605782583486871e-06, 2.704156713662087e-06, 2.777826694000396e-06, 2.9182419893913902e-06, 6.766480510123074e-06, 7.316928076761542e-06, 7.808898772054818e-06, 7.949842256493866e-06, 1.2430929018591996e-05, 1.1129822269140277e-05, 8.550151505914982e-06, 5.519110800378257e-06, 5.445627266453812e-06, 3.930169896193547e-06, 2.6504228571866406e-06, 2.3208285711007193e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 5.2933459755877266e-05, "first_to_last_ratio": 0.891044777878887, "imbalance_abs_log_ratio": 0.11536059701777933, "last_quartile_mean": 5.940606024523731e-05, "mean": 5.372011116833164e-05, "normalized": [ 0.9420147388585198, 1.0659988587532743, 0.963709291633319, 0.8975605176672462, 1.125548087353594, 1.0475717757518221, 0.976732104642592, 0.8637164816972029, 0.846111937771526, 0.7632509115694202, 0.7157482979891343, 0.6814489061203993, 0.8446272229108893, 0.7783612239209267, 0.7274403936561564, 0.6746000798112963, 0.7942333296252567, 0.7353925302421018, 0.7290165995666343, 0.7029916284483115, 1.7662286900182942, 1.6280111090772478, 1.5041953435495587, 1.3787382922454425, 2.2602767022280488, 1.8755197942670356, 1.4006075651598349, 0.9397341669899374, 0.8802439910879608, 0.625791060722137, 0.4518837761038867, 0.41269459056099206 ], "peak_layer": 25, "peak_normalized": 2.2602767022280488, "population_cv": 0.41623158101695057, "rest_mean": 4.668188099727828e-05, "spike_contrast": 1.9649284076056786, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.17265540920198e-05, "top_five_layers": [ 25, 26, 21, 22, 23 ], "top_five_spike_overlap": 4 }, "values": [ 5.060513649368659e-05, 5.726557719754055e-05, 5.177057028049603e-05, 4.821705078938976e-05, 6.046456837793812e-05, 5.6275672250194475e-05, 5.2470157243078575e-05, 4.6398945414694026e-05, 4.5453227357938886e-05, 4.1001923818839714e-05, 3.845007813652046e-05, 3.660751099232584e-05, 4.537346831057221e-05, 4.181365147815086e-05, 3.907817881554365e-05, 3.6239591281628236e-05, 4.266630276106298e-05, 3.9505368476966396e-05, 3.916285277227871e-05, 3.776478843064979e-05, 9.488200157647952e-05, 8.745693776290864e-05, 8.08055410743691e-05, 7.406597433146089e-05, 0.00012142231571488082, 0.00010075313184643164, 7.524079410359263e-05, 5.0482623919378966e-05, 4.728680505650118e-05, 3.3617565350141376e-05, 2.427524668746628e-05, 2.2169999283505604e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 3.8164853322086856e-05, "first_to_last_ratio": 1.098225537051156, "imbalance_abs_log_ratio": 0.09369572914459387, "last_quartile_mean": 3.475138032627001e-05, "mean": 3.635320979356038e-05, "normalized": [ 1.0100545237401017, 1.1277941417294506, 1.0339617898657618, 0.973999682862056, 1.2083764133743773, 1.1263702014298165, 1.0176352620616909, 0.9004838738201283, 0.9060501404127387, 0.8205445719148284, 0.7643960510337217, 0.7243754628674577, 0.9146516244688326, 0.8385494258672699, 0.7801519619575821, 0.7183706258648994, 0.8752495381665008, 0.8055207958726972, 0.7929505623350485, 0.7733149169620687, 1.889825465144903, 1.6608023458638368, 1.4702267154771183, 1.2188474632943946, 2.049378030759128, 1.5903594818483247, 1.190544785977468, 0.8755521592548481, 0.6990470084527223, 0.49452078606865957, 0.38563108363402304, 0.3624631076175449 ], "peak_layer": 25, "peak_normalized": 2.049378030759128, "population_cv": 0.3831599192278561, "rest_mean": 3.192474253376497e-05, "spike_contrast": 1.887781332385522, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.026693299645558e-05, "top_five_layers": [ 25, 21, 22, 26, 23 ], "top_five_spike_overlap": 4 }, "values": [ 3.6718724004458636e-05, 4.099893703823909e-05, 3.758782986551523e-05, 3.54080148099456e-05, 4.392836126498878e-05, 4.0947172237792984e-05, 3.699430817505345e-05, 3.273547918070108e-05, 3.293783083790913e-05, 2.9829428967786953e-05, 2.778825000859797e-05, 2.6333373170928098e-05, 3.325052239233628e-05, 3.0483963200822473e-05, 2.8361027943901718e-05, 2.6115078071597964e-05, 3.181813008268364e-05, 2.9283266485435888e-05, 2.8826298148487695e-05, 2.8112479412811808e-05, 6.870122160762548e-05, 6.037549610482529e-05, 5.3447460231836885e-05, 4.4309017539490014e-05, 7.450146949850023e-05, 5.781467189081013e-05, 4.3280124373268336e-05, 3.182913133059628e-05, 2.541260255384259e-05, 1.7977417883230373e-05, 1.401892768626567e-05, 1.3176697393646464e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00014837673916190397, "first_to_last_ratio": 0.731612643476868, "imbalance_abs_log_ratio": 0.31250408066168484, "last_quartile_mean": 0.0002028077842624043, "mean": 0.0001626056878194504, "normalized": [ 0.8551186661835177, 0.9765777145868246, 0.8835028575047086, 0.8122059786115863, 1.0442675271893398, 0.9707723657486443, 0.9326015035742744, 0.8249065998812904, 0.7951669620747739, 0.7149157330756515, 0.6760616438979659, 0.64634428961019, 0.7767825221614669, 0.7268909765897233, 0.683656028170462, 0.6339954615620502, 0.7327199513272913, 0.6687002074137507, 0.6699574811277571, 0.639760091520604, 1.6760121743744965, 1.5992113558444732, 1.557153317284483, 1.5248248536541513, 2.469284500404055, 2.118908295850593, 1.6203286142789997, 1.055231913330178, 1.0636268934205615, 0.7272180700073825, 0.48325376647049667, 0.4400416832682564 ], "peak_layer": 25, "peak_normalized": 2.469284500404055, "population_cv": 0.47102059604449703, "rest_mean": 0.00013956093149587373, "spike_contrast": 2.056788879882556, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0002870473719667643, "top_five_layers": [ 25, 26, 21, 27, 22 ], "top_five_spike_overlap": 3 }, "values": [ 0.0001390471588820219, 0.00015879709098953754, 0.00014366258983500302, 0.0001320693118032068, 0.0001698038395261392, 0.00015785310824867338, 0.00015164630895014852, 0.0001341345050605014, 0.00012929867079947144, 0.0001162493645097129, 0.00010993146861437708, 0.00010509925778023899, 0.0001263092563021928, 0.00011819660721812397, 0.00011116635869257152, 0.0001030912681017071, 0.00011914443166460842, 0.00010873445717152208, 0.00010893889702856541, 0.00010402862972114235, 0.00027252911240793765, 0.0002600408624857664, 0.0002532019861973822, 0.00024794519413262606, 0.00040151970461010933, 0.0003445465408731252, 0.0002634746488183737, 0.00017158671107608825, 0.00017295178258791566, 0.00011824979446828365, 7.857981108827516e-05, 7.155328057706356e-05 ] } } }, "post_mlp_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "89f2226456bfe96e245d2aad0f699d32c676f9f53099789cd870253d002402f5", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.16915453970432281, 0.374446302652359, 0.39966297149658203, 0.4194087088108063, 0.09892481565475464, 0.11664740741252899, 0.13951337337493896, 0.18593810498714447, 0.06607726216316223, 0.10425819456577301, 0.14551182091236115, 0.19865906238555908, 0.06484340131282806, 0.10430748760700226, 0.14720691740512848, 0.20439592003822327, 0.10052042454481125, 0.14085541665554047, 0.1745898574590683, 0.2420409768819809, 0.049456026405096054, 0.08509506285190582, 0.11783231794834137, 0.15649692714214325, 0.05533855780959129, 0.08507220447063446, 0.11942370235919952, 0.1559232622385025, 0.06628894805908203, 0.12384890764951706, 0.19928614795207977, 0.30097496509552 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.7713038232614053e-05, "first_to_last_ratio": 0.9452329282464459, "imbalance_abs_log_ratio": 0.05632389694393667, "last_quartile_mean": 1.873933683782525e-05, "mean": 1.785050648095421e-05, "normalized": [ 0.9253720171215417, 1.0755510619639979, 0.9670347617832037, 0.8963865046139576, 1.1468829747999003, 1.0665407472588135, 0.9882554988930321, 0.8723677622432784, 0.8003584606692485, 0.7473858970880981, 0.7114418520855136, 0.6868659644834372, 0.8206718178590082, 0.7666623578348617, 0.7137988286199837, 0.6764206250464467, 0.7833051291297598, 0.7287392881337927, 0.7232099719978157, 0.7295468557531476, 1.8338182321516228, 1.711749265946987, 1.6511533659404045, 1.5781367192936766, 2.3061683540101057, 1.7326924239785666, 1.1180930108426685, 0.8849190444191201, 0.8368592432981226, 0.556884466029555, 0.49886345843807234, 0.4638640382722599 ], "peak_layer": 25, "peak_normalized": 2.3061683540101057, "population_cv": 0.42887089482604773, "rest_mean": 1.51524183348994e-05, "spike_contrast": 2.1396045009514593, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.242018246965017e-05, "top_five_layers": [ 25, 21, 26, 22, 23 ], "top_five_spike_overlap": 4 }, "values": [ 1.651835918892175e-05, 1.9199131202185526e-05, 1.7262060282519087e-05, 1.600095311005134e-05, 2.047244197456166e-05, 1.9038292521145195e-05, 1.7640861187828705e-05, 1.5572206393699162e-05, 1.4286803889262956e-05, 1.334121679974487e-05, 1.2699597391474526e-05, 1.226090535055846e-05, 1.4649407603428699e-05, 1.3685311387234833e-05, 1.2741670616378542e-05, 1.2074450751242694e-05, 1.3982393284095451e-05, 1.3008365385758225e-05, 1.2909664292237721e-05, 1.3022780876781326e-05, 3.273458423791453e-05, 3.05555913655553e-05, 2.9473923859768547e-05, 2.817053973558359e-05, 4.116627314942889e-05, 3.092943734372966e-05, 1.9958526536356658e-05, 1.579625313752331e-05, 1.4938361346139573e-05, 9.940669770003296e-06, 8.90496539796004e-06, 8.280208021460567e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 7.054710840748157e-05, "first_to_last_ratio": 0.9422223014756168, "imbalance_abs_log_ratio": 0.059514043419138664, "last_quartile_mean": 7.487310404030723e-05, "mean": 7.126564014470205e-05, "normalized": [ 0.9379999846855562, 1.0868211553726899, 0.9748707333321422, 0.8974108890800359, 1.1351222985959208, 1.0472097338604904, 0.9740426303067738, 0.8658630327193092, 0.7997897837345292, 0.7493484980359226, 0.7134681855191725, 0.6873398045366488, 0.8313442253060974, 0.7699999234277809, 0.7167729914468831, 0.678050930735323, 0.7908526827076957, 0.731113487685911, 0.7234361528075204, 0.7232407405044067, 1.8221819509069979, 1.7163950820216087, 1.6472035059866714, 1.5751619119572624, 2.3017700944645942, 1.7269185552353703, 1.1110479929145174, 0.8827626235684185, 0.8409233588659144, 0.5633975604090998, 0.5064015396120851, 0.47173795965665016 ], "peak_layer": 25, "peak_normalized": 2.3017700944645942, "population_cv": 0.4259330183752373, "rest_mean": 6.054223976442935e-05, "spike_contrast": 2.133584794695151, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001291720021981746, "top_five_layers": [ 25, 21, 26, 22, 23 ], "top_five_spike_overlap": 4 }, "values": [ 6.684716936433688e-05, 7.745300536043942e-05, 6.947478686925024e-05, 6.395456148311496e-05, 8.089521725196391e-05, 7.463007204933092e-05, 6.94157715770416e-05, 6.170628330437467e-05, 5.6997530919034034e-05, 5.340280040400103e-05, 5.084576696390286e-05, 4.898371116723865e-05, 5.924627839704044e-05, 5.4874537454452366e-05, 5.108128607389517e-05, 4.832173362956382e-05, 5.6360622693318874e-05, 5.210327071836218e-05, 5.155614053364843e-05, 5.1542214350774884e-05, 0.00012985896319150925, 0.00012231999426148832, 0.00011738901230273768, 0.00011225492198718712, 0.0001640371192479506, 0.00012306995631661266, 7.917954644653946e-05, 6.291064346441999e-05, 5.9928941482212394e-05, 4.015088779851794e-05, 3.608902989071794e-05, 3.36187076754868e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.06256188219413161, "first_to_last_ratio": 0.9422223137673303, "imbalance_abs_log_ratio": 0.05951403037368916, "last_quartile_mean": 0.06639821757562459, "mean": 0.06319908314617351, "normalized": [ 0.9379999722054387, 1.0868212304924176, 0.9748707426028633, 0.8974108719586117, 1.1351223240365425, 1.0472097384089414, 0.9740426197757958, 0.8658629820174374, 0.7997898149816961, 0.7493485346908154, 0.7134681635915892, 0.6873398244345456, 0.831344188908277, 0.7699999103202189, 0.7167729920080155, 0.6780509258863604, 0.7908527021144804, 0.7311135046411672, 0.7234361711931336, 0.7232407085057854, 1.8221820403015225, 1.7163949931582028, 1.6472035596497798, 1.5751618840472554, 2.3017702566825893, 1.7269185014731279, 1.1110478756750248, 0.8827626057999336, 0.8409233342213907, 0.5633975592074006, 0.5064015001783359, 0.4717379668313034 ], "peak_layer": 25, "peak_normalized": 2.3017702566825893, "population_cv": 0.42593303449142766, "rest_mean": 0.053689463891916804, "spike_contrast": 2.1335848566073294, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.1145510271191597, "top_five_layers": [ 25, 21, 26, 22, 23 ], "top_five_spike_overlap": 4 }, "values": [ 0.05928073823451996, 0.0686861053109169, 0.06161093711853027, 0.05671554431319237, 0.07173869013786316, 0.0661826953291893, 0.06155860051512718, 0.05472174659371376, 0.050545983016490936, 0.04735814034938812, 0.045090533792972565, 0.04343924671411514, 0.052540190517902374, 0.04866328835487366, 0.04529939591884613, 0.04285219684243202, 0.049981165677309036, 0.04620570316910744, 0.045720502734184265, 0.045708149671554565, 0.11516023427248001, 0.10847458988428116, 0.1041017547249794, 0.09954878687858582, 0.1454697698354721, 0.10913966596126556, 0.07021720707416534, 0.05578978732228279, 0.05314558371901512, 0.03560620918869972, 0.032004110515117645, 0.029813406988978386 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 2.776905176915534e-06, "first_to_last_ratio": 0.5399723704789311, "imbalance_abs_log_ratio": 0.6162373065125873, "last_quartile_mean": 5.1426801235265884e-06, "mean": 3.843909098577569e-06, "normalized": [ 0.8227615185915796, 0.7374788170445089, 0.6983207015909613, 0.6868446844320057, 0.756346487108986, 0.7120515807369246, 0.6881664286896052, 0.6773655687898235, 0.6118886829293165, 0.5947080187375955, 0.5996884717696543, 0.6093446875374688, 0.6648157712098252, 0.6563350175985116, 0.6542038417713892, 0.6478054047007011, 0.672612021596751, 0.6821976694120341, 0.6920922664444772, 0.7477555930721791, 1.7529830635139374, 1.885124955850666, 2.0194110965354803, 2.0266763423039023, 2.921968466121852, 2.254610710677251, 1.5181164733225043, 1.137253129730101, 1.013109845672679, 0.6852011550429971, 0.6123113808243626, 0.5604501466399687 ], "peak_layer": 25, "peak_normalized": 2.921968466121852, "population_cv": 0.6006044924328552, "rest_mean": 3.04577633869485e-06, "spike_contrast": 2.6770928312609654, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 8.153826001944253e-06, "top_five_layers": [ 25, 26, 24, 23, 22 ], "top_five_spike_overlap": 4 }, "values": [ 3.1626204872736707e-06, 2.83480153484561e-06, 2.6842812985705677e-06, 2.6401685317978263e-06, 2.9073271434754133e-06, 2.737061549851205e-06, 2.6452491965756053e-06, 2.603731672934373e-06, 2.352044475628645e-06, 2.286003564222483e-06, 2.3051479729474522e-06, 2.3422655885951826e-06, 2.5554913918313105e-06, 2.522892145861988e-06, 2.5147000997094437e-06, 2.4901050892367493e-06, 2.585459469628404e-06, 2.6223058284813305e-06, 2.660339760041097e-06, 2.8743045277224155e-06, 6.738307547493605e-06, 7.246248969750013e-06, 7.762432687741239e-06, 7.790359632053878e-06, 1.1231781172682531e-05, 8.666518624522723e-06, 5.835501724504866e-06, 4.371497652755352e-06, 3.894302153639728e-06, 2.633850954225636e-06, 2.3536692879133625e-06, 2.1543194179685088e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 5.252136588751455e-05, "first_to_last_ratio": 1.1102813332094579, "imbalance_abs_log_ratio": 0.10461343655222459, "last_quartile_mean": 4.730455634671671e-05, "mean": 5.009007662692966e-05, "normalized": [ 1.0081011961376989, 1.141933581620483, 1.034425625713019, 0.9611669835948158, 1.2029266106613417, 1.11218310944662, 1.0229584202988176, 0.9046112070036723, 0.8482367843591191, 0.7909030814075713, 0.7457002652334379, 0.7135836215633923, 0.8839802890747728, 0.810254426316435, 0.7458780603721056, 0.6981659940975514, 0.8462495169636021, 0.7745598733669014, 0.7649147049803782, 0.7696407292729622, 1.8857905296074395, 1.7268522794335537, 1.5952973857638175, 1.4565674995495628, 2.107405488113277, 1.5479049585437064, 1.0247860497389458, 0.8317047416123479, 0.6769844777429004, 0.4921280831267444, 0.44719726155813005, 0.42700716372487746 ], "peak_layer": 25, "peak_normalized": 2.107405488113277, "population_cv": 0.3998097251679885, "rest_mean": 4.3092468466191485e-05, "spike_contrast": 2.039269594496785, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 8.78771606949158e-05, "top_five_layers": [ 25, 21, 22, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 5.049586616223678e-05, 5.719954060623422e-05, 5.1814458856824785e-05, 4.8144927859539166e-05, 6.0254686104599386e-05, 5.570933717535809e-05, 5.124006565893069e-05, 4.531204467639327e-05, 4.248824552632868e-05, 3.961639595218003e-05, 3.7352183426264673e-05, 3.57434582838323e-05, 4.42786404164508e-05, 4.058570630149916e-05, 3.7361089198384434e-05, 3.497118814266287e-05, 4.238870315020904e-05, 3.879776340909302e-05, 3.8314636185532436e-05, 3.85513631044887e-05, 9.445939213037491e-05, 8.649816300021484e-05, 7.990856829565018e-05, 7.295957766473293e-05, 0.00010556010238360614, 7.753467798465863e-05, 5.1331611757632345e-05, 4.166015423834324e-05, 3.3910204365383834e-05, 2.4650733394082636e-05, 2.240014509879984e-05, 2.138882155122701e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 3.791948302023229e-05, "first_to_last_ratio": 1.335279249189923, "imbalance_abs_log_ratio": 0.2891404454012476, "last_quartile_mean": 2.839816693267494e-05, "mean": 3.441257513259188e-05, "normalized": [ 1.0637305233274608, 1.1892600492800354, 1.092852684922268, 1.0286725887405554, 1.270208587876217, 1.1768241883383728, 1.0545530579682312, 0.9391602693726474, 0.902169201905969, 0.8434708261231891, 0.7879480698622023, 0.7485805471526681, 0.945925705869091, 0.8630611604746478, 0.7894625651939113, 0.734847385536493, 0.9185208039537673, 0.8377973358979465, 0.8203131924227713, 0.8398411540082306, 1.994321197517992, 1.7405192239406022, 1.5379838582188798, 1.2781642870377248, 1.8396980534397966, 1.3127174250862446, 0.9488406283723168, 0.8099074064395626, 0.5240784975531838, 0.4172921463582769, 0.3838650481691525, 0.3654123296395926 ], "peak_layer": 21, "peak_normalized": 1.994321197517992, "population_cv": 0.38011441543341845, "rest_mean": 3.0091010018993444e-05, "spike_contrast": 1.9191455092259204, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.7749026746023445e-05, "top_five_layers": [ 21, 25, 22, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 3.6605706554837525e-05, 4.092550079803914e-05, 3.760787512874231e-05, 3.539927274687216e-05, 4.3711148464353755e-05, 4.049755079904571e-05, 3.628988633863628e-05, 3.231892333133146e-05, 3.1045965442899615e-05, 2.902600317611359e-05, 2.7115322154713795e-05, 2.576058432168793e-05, 3.25517394230701e-05, 2.9700157028855756e-05, 2.716743983910419e-05, 2.5287990865763277e-05, 3.160866617690772e-05, 2.88307637674734e-05, 2.8229089366504923e-05, 2.8901096811750904e-05, 6.862972804810852e-05, 5.989574856357649e-05, 5.2925985073670745e-05, 4.398492455948144e-05, 6.330874748528004e-05, 4.517398701864295e-05, 3.265204941271804e-05, 2.7870999474544078e-05, 1.8034890672424808e-05, 1.436009733879473e-05, 1.3209784810896963e-05, 1.2574779248097911e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00014700513384013902, "first_to_last_ratio": 0.9343845740629444, "imbalance_abs_log_ratio": 0.06786717596681079, "last_quartile_mean": 0.00015732829706394114, "mean": 0.00014926616859156638, "normalized": [ 0.9228700424167972, 1.0637586591577493, 0.9636267848293926, 0.8840279978743344, 1.1355311673134947, 1.0486293288233295, 0.9950427458249301, 0.8653319090736888, 0.8092678688642537, 0.7469061884809476, 0.7151122111416504, 0.6895216014283412, 0.8218693398084698, 0.7590237692582388, 0.7048522967551623, 0.6634902894888643, 0.7942662987925022, 0.7202477252482649, 0.712443235398747, 0.7063868335079536, 1.8191425526685712, 1.7147632467309508, 1.6770082124355337, 1.6347856147696127, 2.3106154691014655, 1.7765767912295918, 1.1539182137389712, 0.9305932745944843, 0.7925714889593393, 0.5314123051314781, 0.4794085650363105, 0.4569979721165786 ], "peak_layer": 25, "peak_normalized": 2.3106154691014655, "population_cv": 0.438115417611488, "rest_mean": 0.00012628849341396104, "spike_contrast": 2.164453840261089, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00027334561455063524, "top_five_layers": [ 25, 21, 26, 22, 23 ], "top_five_spike_overlap": 4 }, "values": [ 0.00013775327533949167, 0.00015878317935857922, 0.00014383687812369317, 0.0001319554721703753, 0.00016949638666119426, 0.0001565248821862042, 0.00014852621825411916, 0.00012916477862745523, 0.00012079631414962932, 0.00011148782505188137, 0.00010674205987015739, 0.00010292224760632962, 0.0001226772874360904, 0.00011329656990710646, 0.00010521060175960883, 9.903665340971202e-05, 0.00011855708726216108, 0.00010750861838459969, 0.00010634367208695039, 0.00010543965618126094, 0.00027153643895871937, 0.00025595613988116384, 0.00025032059056684375, 0.00024401818518526852, 0.0003448967181611806, 0.0002651828108355403, 0.0001722409506328404, 0.00013890609261579812, 0.00011830410949187353, 7.932187872938812e-05, 7.155947969295084e-05, 6.821433635195717e-05 ] } } }, "pre_attention_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "ee695e78937de01b7128d3866b8fc5dd328ebdf2c159be37918d4f02952d5248", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.0626930519938469, 0.09030644595623016, 0.20229491591453552, 0.20300470292568207, 0.19049881398677826, 0.12695962190628052, 0.12132706493139267, 0.11585728824138641, 0.13427142798900604, 0.10485932976007462, 0.10419609397649765, 0.10852954536676407, 0.1286265105009079, 0.09096142649650574, 0.10080574452877045, 0.10566592961549759, 0.1287592649459839, 0.09650235623121262, 0.1065765842795372, 0.11446042358875275, 0.12395420670509338, 0.07221997529268265, 0.07677769660949707, 0.08416415005922318, 0.09094838052988052, 0.059439923614263535, 0.062182098627090454, 0.06896103918552399, 0.0776243805885315, 0.06285123527050018, 0.07231374084949493, 0.08016443997621536 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.0671103041204333e-05, "first_to_last_ratio": 1.2377067286720753, "imbalance_abs_log_ratio": 0.21326025498121487, "last_quartile_mean": 8.621673288189413e-06, "mean": 1.1472239769716452e-05, "normalized": [ 0.5816177525952073, 1.1596012491989445, 0.813713341294041, 0.7364771962160369, 0.46496338346382765, 0.986929297609102, 1.2878402552411385, 1.4101964471424349, 0.9365491642923311, 0.7670317583190978, 0.5930323781196577, 0.5722087778348641, 0.523214857666273, 0.943216750041352, 0.8107394699369442, 0.7600683868472863, 0.8729737849132535, 1.2683185560926813, 1.2685391071194312, 1.493040549995506, 1.244430073459163, 2.3990037581493366, 2.008849784607392, 2.0852451086251125, 1.3974740935584038, 1.4178123547397599, 0.9055615072003649, 0.6569632790945154, 0.5381869982066746, 0.3288484978780576, 0.36755213105531265, 0.39979994948649655 ], "peak_layer": 22, "peak_normalized": 2.3990037581493366, "population_cv": 0.5022446964088553, "rest_mean": 9.715286296341219e-06, "spike_contrast": 2.157403074558511, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.0959788525942712e-05, "top_five_layers": [ 22, 24, 23, 20, 26 ], "top_five_spike_overlap": 3 }, "values": [ 6.672458312095841e-06, 1.3303223568073008e-05, 9.335114555142354e-06, 8.449042979918886e-06, 5.334171419235645e-06, 1.1322289537929464e-05, 1.4774412193219177e-05, 1.6178111764020287e-05, 1.074431656888919e-05, 8.799572242423892e-06, 6.803409632993862e-06, 6.564516297657974e-06, 6.00244629822555e-06, 1.08208087112871e-05, 9.300997589889448e-06, 8.719686775293667e-06, 1.0014964573201723e-05, 1.4550454579875804e-05, 1.4552984794136137e-05, 1.7128519175457768e-05, 1.4276400179369375e-05, 2.752194632194005e-05, 2.3046006390359253e-05, 2.3922431864775717e-05, 1.603215787326917e-05, 1.6265483282040805e-05, 1.0388818736828398e-05, 7.536840257671429e-06, 6.174210284370929e-06, 3.7726288155681686e-06, 4.216646175336791e-06, 4.586600880429614e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 4.3214643710598466e-05, "first_to_last_ratio": 1.2687877031395332, "imbalance_abs_log_ratio": 0.23806188012664942, "last_quartile_mean": 3.405979077797383e-05, "mean": 4.606112958072117e-05, "normalized": [ 0.6686114084824253, 1.1708993894543316, 0.8100123699772465, 0.7558132647963478, 0.45989373273618633, 1.004200404735703, 1.246247933650168, 1.3899374641591196, 0.9206895925872056, 0.7588538170236042, 0.6192417478495157, 0.5712724298664037, 0.5294861815019832, 0.9734505229621165, 0.8166506891748119, 0.7539440087617871, 0.8869471003792804, 1.3422657880979343, 1.2689658670070303, 1.5050650594305646, 1.2577863463805077, 2.3371867310410783, 1.9873258500457947, 2.0496716638165786, 1.3660488664618566, 1.4348354666063403, 0.8931286693487702, 0.5841747352672472, 0.5295934779199049, 0.33608077479298226, 0.3678770039678387, 0.403841641717335 ], "peak_layer": 22, "peak_normalized": 2.3371867310410783, "population_cv": 0.49249397088938496, "rest_mean": 3.924063727296285e-05, "spike_contrast": 2.1123965817886776, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 8.28917880426161e-05, "top_five_layers": [ 22, 24, 23, 20, 26 ], "top_five_spike_overlap": 3 }, "values": [ 3.0796996725257486e-05, 5.3932948503643274e-05, 3.731008473550901e-05, 3.48136127286125e-05, 2.118322481692303e-05, 4.6254604967543855e-05, 5.740358756156638e-05, 6.40220896457322e-05, 4.240800262778066e-05, 3.495366399874911e-05, 2.852297438948881e-05, 2.6313453417969868e-05, 2.43887316173641e-05, 4.483823067857884e-05, 3.761585321626626e-05, 3.472751268418506e-05, 4.085378532181494e-05, 6.182627839734778e-05, 5.8450001233723015e-05, 6.932499672984704e-05, 5.793505988549441e-05, 0.00010765346087282524, 9.15384734980762e-05, 9.44101921049878e-05, 6.292175385169685e-05, 6.609014235436916e-05, 4.113851537113078e-05, 2.690774817892816e-05, 2.4393673811573535e-05, 1.5480260117328726e-05, 1.6944830349530093e-05, 1.860140218923334e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.038323177956044674, "first_to_last_ratio": 1.2687877055192325, "imbalance_abs_log_ratio": 0.23806188200221876, "last_quartile_mean": 0.030204562819562852, "mean": 0.040847470314474776, "normalized": [ 0.6686114117167236, 1.1708993321281835, 0.8100123530932102, 0.7558133184601483, 0.45989372576555726, 1.0042003703215474, 1.2462479716713744, 1.3899374408657348, 0.920689602174684, 0.7588537904810377, 0.6192417375875287, 0.5712724398364178, 0.5294861389546389, 0.9734504580194555, 0.816650711685484, 0.7539440371554658, 0.8869471434646419, 1.342265819994993, 1.2689658978049303, 1.5050650687487552, 1.2577864163866102, 2.3371866790111886, 1.9873258741531445, 2.04967167018573, 1.3660488712161196, 1.4348353953876214, 0.8931286811335798, 0.5841747353198764, 0.5295934813836201, 0.3360807775376089, 0.3678770049221929, 0.4038416434321957 ], "peak_layer": 22, "peak_normalized": 2.3371866790111886, "population_cv": 0.492493968443134, "rest_mean": 0.03479898945186977, "spike_contrast": 2.11239659916596, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.07350926697254181, "top_five_layers": [ 22, 24, 23, 20, 26 ], "top_five_spike_overlap": 3 }, "values": [ 0.027311084792017937, 0.047828275710344315, 0.033086955547332764, 0.03087306208908558, 0.018785495311021805, 0.04101904481649399, 0.05090607702732086, 0.05677542835474014, 0.037607841193675995, 0.03099725767970085, 0.025294458493590355, 0.023335034027695656, 0.02162816934287548, 0.039762988686561584, 0.03335811570286751, 0.030796706676483154, 0.03622954711318016, 0.05482816323637962, 0.051834046840667725, 0.06147810071706772, 0.051377393305301666, 0.09546816349029541, 0.0811772346496582, 0.08372390270233154, 0.05579964071512222, 0.05860939621925354, 0.03648204728960991, 0.02386206015944481, 0.02163255400955677, 0.013728049583733082, 0.015026845037937164, 0.01649590954184532 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 3.1034694529807894e-06, "first_to_last_ratio": 0.5113504871686874, "imbalance_abs_log_ratio": 0.6707000390197304, "last_quartile_mean": 6.069162992616839e-06, "mean": 3.990956816579683e-06, "normalized": [ 2.672943607274198, 0.9706530559365012, 0.27257228441169956, 0.3462760793359085, 0.23689280405120067, 0.5159500655107996, 0.48445456811937376, 0.7212608598523438, 0.486495256118519, 0.46301962439234573, 0.4274079124892293, 0.5373278660997405, 0.36537288309574256, 0.6106341943211929, 0.6871415637755098, 0.7339993927847229, 0.419943497496739, 0.8143551788239324, 0.6830429821663496, 0.9779672636672102, 0.7167588579432884, 1.6372086760286264, 1.9168479372161704, 2.135643144812962, 1.9198360162853563, 2.6617223585427525, 2.155490671534485, 1.6697216972946645, 1.3854998185047516, 1.017315470226234, 0.6626960369518339, 0.693548374935617 ], "peak_layer": 1, "peak_normalized": 2.672943607274198, "population_cv": 0.6975250425992161, "rest_mean": 3.499286511517615e-06, "spike_contrast": 1.8992375851591923, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.645976463914849e-06, "top_five_layers": [ 1, 26, 27, 24, 25 ], "top_five_spike_overlap": 2 }, "values": [ 1.0667602509784047e-05, 3.873834430123679e-06, 1.0878242164835683e-06, 1.381972879244131e-06, 9.454289511268144e-07, 2.0591344309650594e-06, 1.933437260959181e-06, 2.878520945159835e-06, 1.941581558639882e-06, 1.8478913261787966e-06, 1.7057665218089824e-06, 2.1444523099489743e-06, 1.4581873983843252e-06, 2.4370147002628073e-06, 2.7423523079050938e-06, 2.929359879999538e-06, 1.6759763639129233e-06, 3.2500563520443393e-06, 2.7259950456937077e-06, 3.903025117324432e-06, 2.860553649952635e-06, 6.534029125759844e-06, 7.65005734137958e-06, 8.52325956657296e-06, 7.661982635909226e-06, 1.0622818990668748e-05, 8.60247018863447e-06, 6.663787189609138e-06, 5.529469945031451e-06, 4.060062110511353e-06, 2.6447912659932626e-06, 2.7679216145770624e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 2.8506896796898218e-05, "first_to_last_ratio": 1.2872243438766047, "imbalance_abs_log_ratio": 0.2524882287892687, "last_quartile_mean": 2.2146020569380198e-05, "mean": 3.105288445226506e-05, "normalized": [ 0.6009680307518237, 1.1304608114757642, 0.8139010007998151, 0.746074989928323, 0.5098803315456683, 1.0389224196940963, 1.2753531439825625, 1.2285291493905, 0.9277941895132241, 0.7983229355250093, 0.6632306184296753, 0.5939558775377031, 0.5739956558326808, 1.0339913953885804, 0.8411564790744886, 0.7354085599230965, 0.8740993297876409, 1.3568395800398885, 1.2488530766064865, 1.4592387608033297, 1.2437406974512455, 2.4018430106059756, 2.143999527077397, 2.054071421089963, 1.3387496680883801, 1.5018617195063095, 0.8623253230709783, 0.5280245445132281, 0.5366094355698149, 0.33399043682847207, 0.28007544418129077, 0.3237324359865885 ], "peak_layer": 22, "peak_normalized": 2.4018430106059756, "population_cv": 0.5108152499445623, "rest_mean": 2.624267266650417e-05, "spike_contrast": 2.173102900763754, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.7028028095373884e-05, "top_five_layers": [ 22, 23, 24, 26, 20 ], "top_five_spike_overlap": 3 }, "values": [ 1.866179081844166e-05, 3.51040689565707e-05, 2.5273973733419552e-05, 2.3167780454969034e-05, 1.5833255019970238e-05, 3.22615378536284e-05, 3.960339381592348e-05, 3.814937372226268e-05, 2.881068576243706e-05, 2.4790229872451164e-05, 2.0595223759301007e-05, 1.8444043234921992e-05, 1.7824220776674338e-05, 3.210841532563791e-05, 2.612033495097421e-05, 2.283655703649856e-05, 2.7143305487697944e-05, 4.213378269923851e-05, 3.878049028571695e-05, 4.531357262749225e-05, 3.862173616653308e-05, 7.458415348082781e-05, 6.657736958004534e-05, 6.378484249580652e-05, 4.157203875365667e-05, 4.663713843910955e-05, 2.6777688617585227e-05, 1.6396685168729164e-05, 1.6663270798744634e-05, 1.0371366442996077e-05, 8.697150406078435e-06, 1.0052825928141829e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 1.8391206594969844e-05, "first_to_last_ratio": 1.2830797232391786, "imbalance_abs_log_ratio": 0.24926322184733737, "last_quartile_mean": 1.433364292324768e-05, "mean": 2.0513109788566908e-05, "normalized": [ 0.527983566678561, 1.1272924536923925, 0.7987795647783638, 0.747111869839853, 0.5736337225317584, 1.0712609987749933, 1.3131109854158014, 1.013296262781011, 0.9570259188066299, 0.8656479282531988, 0.710650873302505, 0.6102148729934606, 0.6438866580078574, 1.1075703619465511, 0.8657293314285177, 0.7260367828336454, 0.8672789180175136, 1.3429876355730228, 1.248128639518154, 1.3723079420244153, 1.2055810264732993, 2.4345050785279225, 2.3002118305234576, 1.979724869058357, 1.2714241355766338, 1.5886255301006484, 0.8479502749523803, 0.48276671868614346, 0.5446459010540677, 0.34376155251423246, 0.2404843109293846, 0.27038348440526727 ], "peak_layer": 22, "peak_normalized": 2.4345050785279225, "population_cv": 0.5202003586385223, "rest_mean": 1.732867974203179e-05, "spike_contrast": 2.1761053121889566, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.770903203985654e-05, "top_five_layers": [ 22, 23, 24, 26, 20 ], "top_five_spike_overlap": 3 }, "values": [ 1.0830584869836457e-05, 2.3124273866415024e-05, 1.6385452909162268e-05, 1.5325587810366414e-05, 1.1767011528718285e-05, 2.1974894480081275e-05, 2.6935989808407612e-05, 2.078585748677142e-05, 1.963157774298452e-05, 1.7757130990503356e-05, 1.4577659385395236e-05, 1.2517404684331268e-05, 1.3208117707108613e-05, 2.271971243317239e-05, 1.775880082277581e-05, 1.4893272236804478e-05, 1.7790587662602775e-05, 2.75488528131973e-05, 2.560299981269054e-05, 2.815030347846914e-05, 2.4730215955059975e-05, 4.9939269956666976e-05, 4.718449781648815e-05, 4.0610313590150326e-05, 2.6080862880917266e-05, 3.2587649911874905e-05, 1.7394097085343674e-05, 9.903046702675056e-06, 1.117238116421504e-05, 7.051618467812659e-06, 4.933081072522327e-06, 5.546406100620516e-06 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 8.664335746288998e-05, "first_to_last_ratio": 1.3150302755476477, "imbalance_abs_log_ratio": 0.2738596885948477, "last_quartile_mean": 6.588696783182968e-05, "mean": 9.217207826850426e-05, "normalized": [ 0.6697459144565897, 1.1379933166165892, 0.8326241482707284, 0.7504288366756969, 0.4501824837659223, 1.024100584510774, 1.2433354717188694, 1.4117284523460694, 0.8792612192710949, 0.71954046041116, 0.640104535163266, 0.5796766107184564, 0.5017043695776864, 0.9538005678193396, 0.8086199499426691, 0.758430867629162, 0.8725833874815282, 1.3924526909728543, 1.2353567277253656, 1.5531934976179391, 1.3079440764569261, 2.387240360734029, 2.0733498227581437, 2.0979962675480497, 1.3951388220696728, 1.434640137027133, 0.8643630116763587, 0.54283249577027, 0.5293438597028155, 0.3031152077034418, 0.29009773788096604, 0.3590741079804332 ], "peak_layer": 22, "peak_normalized": 2.387240360734029, "population_cv": 0.5183459214835758, "rest_mean": 7.762367379284653e-05, "spike_contrast": 2.1995024725664307, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.000170733462437056, "top_five_layers": [ 22, 24, 23, 20, 26 ], "top_five_spike_overlap": 3 }, "values": [ 6.173187284730375e-05, 0.00010489120904821903, 7.674469816265628e-05, 6.916858546901494e-05, 4.149425512878224e-05, 9.439347923034802e-05, 0.0001146008144132793, 0.0001301219454035163, 8.104333392111585e-05, 6.632153963437304e-05, 5.89997653150931e-05, 5.3429997933562845e-05, 4.6243134420365095e-05, 8.791378058958799e-05, 7.45321813155897e-05, 6.990614929236472e-05, 8.0427824286744e-05, 0.0001283452584175393, 0.0001138653969974257, 0.00014316107262857258, 0.00012055592378601432, 0.0002200369053753093, 0.00019110496214125305, 0.00019337667617946863, 0.00012859284470323473, 0.00013223376299720258, 7.96701351646334e-05, 5.003399928682484e-05, 4.879072366748005e-05, 2.7938758648815565e-05, 2.6738911401480436e-05, 3.309660678496584e-05 ] } } }, "pre_mlp_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "e53fa425d89dfeb0e9395eca9bcd73dc0a55314abf45c5b10b43e2cbfa724345", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.059514161199331284, 0.08157797157764435, 0.14292451739311218, 0.10997853428125381, 0.0991024449467659, 0.1065012738108635, 0.12155740708112717, 0.1095810979604721, 0.06964563578367233, 0.08311934769153595, 0.09363401681184769, 0.0941167026758194, 0.08564268052577972, 0.09493856877088547, 0.09063401818275452, 0.09473779052495956, 0.10505300015211105, 0.09355910867452621, 0.09618847072124481, 0.11416846513748169, 0.047205954790115356, 0.053352247923612595, 0.06512638926506042, 0.07829347252845764, 0.05368548259139061, 0.05310077592730522, 0.061235878616571426, 0.07320034503936768, 0.05304349213838577, 0.0639304369688034, 0.07404901832342148, 0.08157160878181458 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 5.272473146078482e-06, "first_to_last_ratio": 0.1669646650349019, "imbalance_abs_log_ratio": 1.7899730755703422, "last_quartile_mean": 3.15783770474809e-05, "mean": 1.595530137166179e-05, "normalized": [ 0.716682823634198, 0.25791058215561197, 0.17008131743255805, 0.3060540641127617, 0.24058425155991167, 0.1967610015959851, 0.267556601656987, 0.4879913231219985, 0.5611398221932165, 0.41485379706382575, 0.382022228517939, 0.38001507917331073, 0.4360496965819164, 0.45667137975241845, 0.5055532742341784, 0.5047791210385066, 0.38894354912552764, 0.4960347981961209, 0.45227564772917783, 0.4009021097501785, 1.488245761197984, 2.0827795687329855, 2.2449984373053344, 2.327691975656579, 3.0303072825615045, 3.0643757234375792, 2.5970065395074045, 1.717218398854801, 1.948959655958918, 1.5429170173623896, 1.0039709548423947, 0.9286662159557967 ], "peak_layer": 26, "peak_normalized": 3.0643757234375792, "population_cv": 0.8818473213689425, "rest_mean": 1.2306842184746833e-05, "spike_contrast": 2.897329830490231, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.5656980981002564e-05, "top_five_layers": [ 26, 25, 27, 24, 23 ], "top_five_spike_overlap": 3 }, "values": [ 1.1434890438977163e-05, 4.115041065233527e-06, 2.7136986773257377e-06, 4.883184828941012e-06, 3.838594238914084e-06, 3.139381078653969e-06, 4.2689462134148926e-06, 7.786048627167474e-06, 8.95315497473348e-06, 6.619117357331561e-06, 6.0952797866775654e-06, 6.063255113986088e-06, 6.9573043219861574e-06, 7.286329491762444e-06, 8.066254849836696e-06, 8.053903002291918e-06, 6.205711542861536e-06, 7.914384696050547e-06, 7.216194262582576e-06, 6.396513981599128e-06, 2.3745409635012038e-05, 3.3231375709874555e-05, 3.5819626646116376e-05, 3.7139026971999556e-05, 4.8349465942010283e-05, 4.88930381834507e-05, 4.1436022002017125e-05, 2.739873707469087e-05, 3.1096238672034815e-05, 2.4617706003482454e-05, 1.6018659152905457e-05, 1.4817149349255487e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 2.1230628817647812e-05, "first_to_last_ratio": 0.16797785395615525, "imbalance_abs_log_ratio": 1.7839231299576728, "last_quartile_mean": 0.00012638945145226899, "mean": 6.381589872717086e-05, "normalized": [ 0.7281424199404678, 0.2651051856151187, 0.17124706753572805, 0.31262396488547634, 0.2393556532070042, 0.19481965815539662, 0.26533962856965776, 0.4848510039711777, 0.5602333212625794, 0.41459730089549734, 0.3807172013128595, 0.38205208671674695, 0.4341655203405304, 0.45522844545037494, 0.5082288287904371, 0.5066329064774105, 0.3884562416273769, 0.49752489033060104, 0.455889189782189, 0.3909201586944003, 1.4938560264420584, 2.0757021092391703, 2.239094226153435, 2.310958217408292, 3.0391651127242207, 3.057851456112052, 2.6036463220915746, 1.723113473374869, 1.9530759878971604, 1.535910047330995, 1.0077754297446913, 0.9237209179204501 ], "peak_layer": 26, "peak_normalized": 3.057851456112052, "population_cv": 0.881164017304722, "rest_mean": 4.9259313325617564e-05, "spike_contrast": 2.8912595462732855, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00014242145989555864, "top_five_layers": [ 26, 25, 27, 24, 23 ], "top_five_spike_overlap": 3 }, "values": [ 4.646706292987801e-05, 1.6917925677262247e-05, 1.092828551918501e-05, 1.9950379282818176e-05, 1.527469612483401e-05, 1.2432591574906837e-05, 1.693288686510641e-05, 3.0941202567191795e-05, 3.5751792893279344e-05, 2.6457899366505444e-05, 2.4295810362673365e-05, 2.4380997274420224e-05, 2.7706662876880728e-05, 2.9050812372588553e-05, 3.243307946831919e-05, 3.233123425161466e-05, 2.47896841756301e-05, 3.1749998015584424e-05, 2.909297836595215e-05, 2.494692125765141e-05, 9.533176489640027e-05, 0.00013246279559098184, 0.0001428898103768006, 0.00014747587556485087, 0.0001939470530487597, 0.00019513953884597868, 0.00016615403001196682, 0.00010996203491231427, 0.00012463729945011437, 9.801548003451899e-05, 6.431209476431832e-05, 5.894808055018075e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.018827533698640764, "first_to_last_ratio": 0.1679778496508904, "imbalance_abs_log_ratio": 1.783923155587628, "last_quartile_mean": 0.11208343086764216, "mean": 0.05659257710794918, "normalized": [ 0.7281424031993917, 0.26510518254505716, 0.171247068155958, 0.31262395451326686, 0.2393556562808966, 0.19481966326706535, 0.2653396400198464, 0.48485098151731576, 0.5602333301551313, 0.4145973058003345, 0.3807171721544929, 0.38205210031059283, 0.43416554431527965, 0.45522843894661846, 0.5082287960660868, 0.5066329317325279, 0.3884562600736407, 0.49752485045247813, 0.45588920518005277, 0.3909201453214437, 1.4938560613228158, 2.07570200306639, 2.239094128693632, 2.310958216396851, 3.039164997896332, 3.057851556501871, 2.603646456055449, 1.7231133756310277, 1.9530760865279508, 1.5359100997193404, 1.0077754740217346, 0.9237209141591279 ], "peak_layer": 26, "peak_normalized": 3.057851556501871, "population_cv": 0.8811640185585949, "rest_mean": 0.04368365221415405, "spike_contrast": 2.891259433054452, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.1263007715344429, "top_five_layers": [ 26, 25, 27, 24, 23 ], "top_five_spike_overlap": 3 }, "values": [ 0.041207455098629, 0.01500298548489809, 0.009691312909126282, 0.017692195251584053, 0.013545753434300423, 0.01102534681558609, 0.015016254037618637, 0.027438966557383537, 0.03170504793524742, 0.023463129997253418, 0.021545765921473503, 0.02162131294608116, 0.02457054704427719, 0.02576255053281784, 0.028761977329850197, 0.028671663254499435, 0.021983740851283073, 0.028156213462352753, 0.0257999449968338, 0.022123178467154503, 0.08454116433858871, 0.11746932566165924, 0.12671610713005066, 0.1307830810546875, 0.1719941794872284, 0.17305169999599457, 0.14734706282615662, 0.09751542657613754, 0.11052960902452469, 0.08692111074924469, 0.05703261122107506, 0.05227574706077576 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 1.24385834254781e-06, "first_to_last_ratio": 0.1464042990215229, "imbalance_abs_log_ratio": 1.9213833129849534, "last_quartile_mean": 8.49605066832737e-06, "mean": 4.1466366322850945e-06, "normalized": [ 0.6933085010099855, 0.2239485623695535, 0.1728017539163926, 0.3339787160068397, 0.24757049138951356, 0.18291949424651965, 0.21342509691392966, 0.3317914714170583, 0.46081271932435325, 0.36569916363646376, 0.3403400889817028, 0.3769304170327454, 0.38894482670904285, 0.4435379340438518, 0.5142777416496214, 0.49906951291602963, 0.3669249615457762, 0.48455360913895673, 0.44896352231192327, 0.3139124782563191, 1.4448539999504162, 2.137634730668425, 2.2970124758544075, 2.325574471890459, 3.080228195142649, 3.1167622916680373, 2.866809623388693, 1.8063405148246436, 1.967331345356014, 1.5329990412028807, 1.0595029462217507, 0.9612393010150451 ], "peak_layer": 26, "peak_normalized": 3.1167622916680373, "population_cv": 0.9244902776457989, "rest_mean": 3.1813451031397293e-06, "spike_contrast": 2.9419036873532787, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.359210889670066e-06, "top_five_layers": [ 26, 25, 27, 24, 23 ], "top_five_spike_overlap": 3 }, "values": [ 2.8748984277626732e-06, 9.286333124691737e-07, 7.165460829128278e-07, 1.3848883781975019e-06, 1.0265848686685786e-06, 7.58500675601681e-07, 8.849963251122972e-07, 1.3758186696577468e-06, 1.9108229025732726e-06, 1.5164215483309818e-06, 1.4112666804066976e-06, 1.5629934750904795e-06, 1.6128128663694952e-06, 1.8391906451142859e-06, 2.1325229226931697e-06, 2.0694599243142875e-06, 1.5215044868455152e-06, 2.0092677459615516e-06, 1.8616885881783674e-06, 1.3016809816690511e-06, 5.991284524498042e-06, 8.863994480634574e-06, 9.524876077193767e-06, 9.643312296248041e-06, 1.277258706977591e-05, 1.2924080692755524e-05, 1.188761780213099e-05, 7.490237749152584e-06, 8.157808224495966e-06, 6.356789981509792e-06, 4.393373728817096e-06, 3.985910097981105e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 1.4084941881264967e-05, "first_to_last_ratio": 0.18669517876993208, "imbalance_abs_log_ratio": 1.6782780521747884, "last_quartile_mean": 7.544352229160722e-05, "mean": 4.050176815439954e-05, "normalized": [ 0.7915344779501412, 0.2817831558648513, 0.14007785149503885, 0.29371224127148304, 0.2658372659782561, 0.2133737264588881, 0.27620642273665674, 0.5195641393981995, 0.6187289564894211, 0.45622403459932837, 0.41667347605891053, 0.43092941891626996, 0.49533041981051057, 0.5024058898618519, 0.5542331926300959, 0.5487927205616486, 0.42937400385950386, 0.544730039129631, 0.49251816040403174, 0.44087499420301385, 1.60108703338286, 2.178991116463943, 2.3300486721718108, 2.2751950175541693, 3.11852289019737, 2.8925932505101977, 2.4264219789038846, 1.634063108068865, 1.7160455582245682, 1.3816296592996113, 0.903709127363538, 0.8287880001814503 ], "peak_layer": 25, "peak_normalized": 3.11852289019737, "population_cv": 0.8509748945340377, "rest_mean": 3.074557514116384e-05, "spike_contrast": 3.030849479901612, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.318521042587236e-05, "top_five_layers": [ 25, 26, 27, 23, 24 ], "top_five_spike_overlap": 3 }, "values": [ 3.2058545912150294e-05, 1.1412716048653238e-05, 5.673400664818473e-06, 1.1895865100086667e-05, 1.0766879313450772e-05, 8.642013199278153e-06, 1.1186848496436141e-05, 2.1043266315246e-05, 2.5059616746148095e-05, 1.8477880075806752e-05, 1.6876012523425743e-05, 1.7453403415856883e-05, 2.0061757822986692e-05, 2.0348326870589517e-05, 2.2447424271376804e-05, 2.2227075533010066e-05, 1.739040635584388e-05, 2.2062529751565307e-05, 1.9947856344515458e-05, 1.785621680028271e-05, 6.484685582108796e-05, 8.825299300951883e-05, 9.437109110876918e-05, 9.214942110702395e-05, 0.00012630569108296186, 0.00011715514119714499, 9.827438043430448e-05, 6.61824451526627e-05, 6.95028793415986e-05, 5.5958444136194885e-05, 3.6601817555492744e-05, 3.3567379432497546e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 9.137090785316104e-06, "first_to_last_ratio": 0.22155320265305567, "imbalance_abs_log_ratio": 1.5070925257739771, "last_quartile_mean": 4.124106840208697e-05, "mean": 2.4481676760501614e-05, "normalized": [ 0.8565553052482237, 0.29739907235676777, 0.12540001157188194, 0.28302028839791354, 0.30354505967819034, 0.24051580258127941, 0.29387022941549945, 0.5854671007058689, 0.6985845923141693, 0.5036372046061477, 0.45833439377765917, 0.49371911603376384, 0.5769478588642699, 0.5722955251163836, 0.6177957139838762, 0.6229681479662924, 0.48843341165027043, 0.6282216432852616, 0.5469379267318303, 0.5029158998808883, 1.777106418154207, 2.355758794087098, 2.4880944351614778, 2.2059257923225983, 3.0833657967884016, 2.608999610481494, 2.0898098485622, 1.4611372040735673, 1.4775811388886644, 1.2213916513646184, 0.7923796172093504, 0.7418853887398853 ], "peak_layer": 25, "peak_normalized": 3.0833657967884016, "population_cv": 0.8051010713356787, "rest_mean": 1.8215953163976168e-05, "spike_contrast": 3.2014017414727314, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.8316584181739016e-05, "top_five_layers": [ 25, 26, 23, 22, 24 ], "top_five_spike_overlap": 4 }, "values": [ 2.0969910110579804e-05, 7.28082795831142e-06, 3.0700025490659755e-06, 6.928811217221664e-06, 7.431292033288628e-06, 5.888230134587502e-06, 7.194435966084711e-06, 1.433321631338913e-05, 1.710252217890229e-05, 1.2329883247730322e-05, 1.1220794476685114e-05, 1.2087071809219196e-05, 1.4124650988378562e-05, 1.4010754057380836e-05, 1.5124674973776564e-05, 1.525130483059911e-05, 1.1957668903050944e-05, 1.5379919204860926e-05, 1.3389957530307584e-05, 1.2312224498600699e-05, 4.350654489826411e-05, 5.767292532254942e-05, 6.0912723711226135e-05, 5.4004762205295265e-05, 7.548596477136016e-05, 6.387268513208255e-05, 5.116204920341261e-05, 3.577108873287216e-05, 3.617366382968612e-05, 2.9901715606683865e-05, 1.9398781660129316e-05, 1.8162598280468956e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 4.25474665917136e-05, "first_to_last_ratio": 0.15783400092853345, "imbalance_abs_log_ratio": 1.846211425285223, "last_quartile_mean": 0.0002695709817999159, "mean": 0.00013444573704646245, "normalized": [ 0.7168928805086953, 0.2613802751409249, 0.13375022258582456, 0.27939894779533553, 0.2340058692654982, 0.18774842881108625, 0.2595070287116215, 0.459042084353912, 0.562580214679595, 0.4240044099309199, 0.38140052790307216, 0.39200725680386445, 0.42435747688951403, 0.44495506988594835, 0.5116524706293077, 0.49695224314665526, 0.3939570804475165, 0.4787773042686179, 0.44184847030116386, 0.38471383184842534, 1.4659742760030587, 2.023682826895762, 2.239802234202718, 2.3611754412819796, 3.150563240019213, 3.0813147086155035, 2.7856825008007937, 1.8800120667796993, 1.8586107106825538, 1.459005478906468, 0.9491043805815779, 0.8761400413231738 ], "peak_layer": 25, "peak_normalized": 3.150563240019213, "population_cv": 0.9058659480325053, "rest_mean": 0.0001033678678747189, "spike_contrast": 2.9241797938622685, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0003022662305738777, "top_five_layers": [ 25, 26, 27, 24, 23 ], "top_five_spike_overlap": 3 }, "values": [ 9.638319170335308e-05, 3.5141463740728796e-05, 1.7982147255679592e-05, 3.7563997466349974e-05, 3.146109156659804e-05, 2.5241975890821777e-05, 3.488961374387145e-05, 6.17162513663061e-05, 7.563651161035523e-05, 5.7005585404112935e-05, 5.12776750838384e-05, 5.270370456855744e-05, 5.705305375158787e-05, 5.9822312323376536e-05, 6.878949352540076e-05, 6.681311060674489e-05, 5.2965850045438856e-05, 6.436956755351275e-05, 5.940464325249195e-05, 5.172313467483036e-05, 0.0001970939920283854, 0.00027207552921026945, 0.00030113186221569777, 0.00031744997249916196, 0.0004235797969158739, 0.00041426962707191706, 0.00037452313699759543, 0.00025275960797443986, 0.00024988228688016534, 0.000196157066966407, 0.00012760303798131645, 0.00011779329361161217 ] } } } }, "selector": { "expected_uniform_count": 4, "identities_unique": true, "identity_order_sha256": "97fd9721a263024f6a746638f32e0ad9a988cd4c111c00e9e178c1022e5c9d5e", "passed": true, "uniform_count": 4, "uniform_indices": [ 41, 43, 45, 47 ], "visit_count": 65, "visits": [ { "branch": "attention", "coefficient": "detached_learned", "group": 1, "index": 0, "kind": "depth", "layer": 1, "offset": 1, "sources": 1, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 1, "index": 1, "kind": "depth", "layer": 1, "offset": 1, "sources": 2, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 1, "index": 2, "kind": "depth", "layer": 2, "offset": 2, "sources": 2, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 1, "index": 3, "kind": "depth", "layer": 2, "offset": 2, "sources": 2, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 1, "index": 4, "kind": "depth", "layer": 3, "offset": 3, "sources": 2, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 1, "index": 5, "kind": "depth", "layer": 3, "offset": 3, "sources": 2, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 1, "index": 6, "kind": "depth", "layer": 4, "offset": 4, "sources": 2, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 1, "index": 7, "kind": "depth", "layer": 4, "offset": 4, "sources": 2, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 2, "index": 8, "kind": "depth", "layer": 5, "offset": 1, "sources": 2, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 2, "index": 9, "kind": "depth", "layer": 5, "offset": 1, "sources": 3, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 2, "index": 10, "kind": "depth", "layer": 6, "offset": 2, "sources": 3, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 2, "index": 11, "kind": "depth", "layer": 6, "offset": 2, "sources": 3, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 2, "index": 12, "kind": "depth", "layer": 7, "offset": 3, "sources": 3, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 2, "index": 13, "kind": "depth", "layer": 7, "offset": 3, "sources": 3, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 2, "index": 14, "kind": "depth", "layer": 8, "offset": 4, "sources": 3, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 2, "index": 15, "kind": "depth", "layer": 8, "offset": 4, "sources": 3, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 3, "index": 16, "kind": "depth", "layer": 9, "offset": 1, "sources": 3, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 3, "index": 17, "kind": "depth", "layer": 9, "offset": 1, "sources": 4, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 3, "index": 18, "kind": "depth", "layer": 10, "offset": 2, "sources": 4, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 3, "index": 19, "kind": "depth", "layer": 10, "offset": 2, "sources": 4, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 3, "index": 20, "kind": "depth", "layer": 11, "offset": 3, "sources": 4, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 3, "index": 21, "kind": "depth", "layer": 11, "offset": 3, "sources": 4, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 3, "index": 22, "kind": "depth", "layer": 12, "offset": 4, "sources": 4, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 3, "index": 23, "kind": "depth", "layer": 12, "offset": 4, "sources": 4, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 4, "index": 24, "kind": "depth", "layer": 13, "offset": 1, "sources": 4, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 4, "index": 25, "kind": "depth", "layer": 13, "offset": 1, "sources": 5, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 4, "index": 26, "kind": "depth", "layer": 14, "offset": 2, "sources": 5, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 4, "index": 27, "kind": "depth", "layer": 14, "offset": 2, "sources": 5, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 4, "index": 28, "kind": "depth", "layer": 15, "offset": 3, "sources": 5, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 4, "index": 29, "kind": "depth", "layer": 15, "offset": 3, "sources": 5, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 4, "index": 30, "kind": "depth", "layer": 16, "offset": 4, "sources": 5, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 4, "index": 31, "kind": "depth", "layer": 16, "offset": 4, "sources": 5, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 5, "index": 32, "kind": "depth", "layer": 17, "offset": 1, "sources": 5, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 5, "index": 33, "kind": "depth", "layer": 17, "offset": 1, "sources": 6, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 5, "index": 34, "kind": "depth", "layer": 18, "offset": 2, "sources": 6, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 5, "index": 35, "kind": "depth", "layer": 18, "offset": 2, "sources": 6, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 5, "index": 36, "kind": "depth", "layer": 19, "offset": 3, "sources": 6, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 5, "index": 37, "kind": "depth", "layer": 19, "offset": 3, "sources": 6, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 5, "index": 38, "kind": "depth", "layer": 20, "offset": 4, "sources": 6, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 5, "index": 39, "kind": "depth", "layer": 20, "offset": 4, "sources": 6, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 6, "index": 40, "kind": "depth", "layer": 21, "offset": 1, "sources": 6, "use_uniform": false }, { "branch": "mlp", "coefficient": "uniform", "group": 6, "index": 41, "kind": "depth", "layer": 21, "offset": 1, "sources": 7, "use_uniform": true }, { "branch": "attention", "coefficient": "detached_learned", "group": 6, "index": 42, "kind": "depth", "layer": 22, "offset": 2, "sources": 7, "use_uniform": false }, { "branch": "mlp", "coefficient": "uniform", "group": 6, "index": 43, "kind": "depth", "layer": 22, "offset": 2, "sources": 7, "use_uniform": true }, { "branch": "attention", "coefficient": "detached_learned", "group": 6, "index": 44, "kind": "depth", "layer": 23, "offset": 3, "sources": 7, "use_uniform": false }, { "branch": "mlp", "coefficient": "uniform", "group": 6, "index": 45, "kind": "depth", "layer": 23, "offset": 3, "sources": 7, "use_uniform": true }, { "branch": "attention", "coefficient": "detached_learned", "group": 6, "index": 46, "kind": "depth", "layer": 24, "offset": 4, "sources": 7, "use_uniform": false }, { "branch": "mlp", "coefficient": "uniform", "group": 6, "index": 47, "kind": "depth", "layer": 24, "offset": 4, "sources": 7, "use_uniform": true }, { "branch": "attention", "coefficient": "detached_learned", "group": 7, "index": 48, "kind": "depth", "layer": 25, "offset": 1, "sources": 7, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 7, "index": 49, "kind": "depth", "layer": 25, "offset": 1, "sources": 8, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 7, "index": 50, "kind": "depth", "layer": 26, "offset": 2, "sources": 8, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 7, "index": 51, "kind": "depth", "layer": 26, "offset": 2, "sources": 8, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 7, "index": 52, "kind": "depth", "layer": 27, "offset": 3, "sources": 8, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 7, "index": 53, "kind": "depth", "layer": 27, "offset": 3, "sources": 8, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 7, "index": 54, "kind": "depth", "layer": 28, "offset": 4, "sources": 8, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 7, "index": 55, "kind": "depth", "layer": 28, "offset": 4, "sources": 8, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 8, "index": 56, "kind": "depth", "layer": 29, "offset": 1, "sources": 8, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 8, "index": 57, "kind": "depth", "layer": 29, "offset": 1, "sources": 9, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 8, "index": 58, "kind": "depth", "layer": 30, "offset": 2, "sources": 9, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 8, "index": 59, "kind": "depth", "layer": 30, "offset": 2, "sources": 9, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 8, "index": 60, "kind": "depth", "layer": 31, "offset": 3, "sources": 9, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 8, "index": 61, "kind": "depth", "layer": 31, "offset": 3, "sources": 9, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 8, "index": 62, "kind": "depth", "layer": 32, "offset": 4, "sources": 9, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 8, "index": 63, "kind": "depth", "layer": 32, "offset": 4, "sources": 9, "use_uniform": false }, { "branch": "output", "coefficient": "detached_learned", "group": null, "index": 64, "kind": "output", "layer": null, "offset": null, "sources": 9, "use_uniform": false } ], "visits_sha256": "185590e26f481634a0c9f936aa00f6f1b54ddf67e90abdc33fea58763842862c" } }, "uniform_group_6_only": { "bits_per_byte": 1.8740129107275643, "forward": { "activation_sha256": { "attention_branch_output": "5e6ecbcd78543fc42d4098a51169dca446d404ce384015643e84d2c78433c949", "mlp_branch_output": "c0b8018c349b1bd09e5d17858eb312a2323da0e92de9651e2bb08ae4adf115c7", "post_attention_state": "90a633f12d984935028fe5d7212be48e7da7b4a0806751939a551938d16dc540", "post_mlp_state": "89f2226456bfe96e245d2aad0f699d32c676f9f53099789cd870253d002402f5", "pre_attention_input": "ee695e78937de01b7128d3866b8fc5dd328ebdf2c159be37918d4f02952d5248", "pre_mlp_input": "e53fa425d89dfeb0e9395eca9bcd73dc0a55314abf45c5b10b43e2cbfa724345" }, "logits_sha256": "bdacba5675f0579cec629bbf01b36c8660ad576cfdf9fef1bea8259eaeac9ec2", "loss_nats": 1.2989667654037476, "mixer_summary_sha256": "eb3ac0f07de9b755d6e67a510ab68d09d9ad666bb7b5d020ba7d8c5055c399b2" }, "loss_nats": 1.2989667654037476, "loss_scale": 1.0, "mixers": [ { "branch": "attention", "entropy_mean": -0.0, "group": 1, "latest_source_mass": 1.0, "layer": 1, "max_source_mass": 1.0, "mixer_index": 0, "normalized_entropy": 1.0, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 1.0, "median": 1.0, "p05": 1.0, "p95": 1.0 } ], "sources": 1 }, { "branch": "mlp", "entropy_mean": 0.22886337339878082, "group": 1, "latest_source_mass": 0.06350618600845337, "layer": 1, "max_source_mass": 0.9364938139915466, "mixer_index": 1, "normalized_entropy": 0.3301800538435402, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.9364938139915466, "median": 0.9452826976776123, "p05": 0.8725647330284119, "p95": 0.9712384939193726 }, { "label": "current_group_partial", "mean": 0.06350618600845337, "median": 0.0547172985970974, "p05": 0.028761589899659157, "p95": 0.12743529677391052 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.603729784488678, "group": 1, "latest_source_mass": 0.4133155345916748, "layer": 2, "max_source_mass": 0.5866844654083252, "mixer_index": 2, "normalized_entropy": 0.8709979661187783, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.5866844654083252, "median": 0.5734395384788513, "p05": 0.29813364148139954, "p95": 0.8843642473220825 }, { "label": "current_group_partial", "mean": 0.4133155345916748, "median": 0.4265604615211487, "p05": 0.11563576757907867, "p95": 0.7018663287162781 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.3967982828617096, "group": 1, "latest_source_mass": 0.14110636711120605, "layer": 2, "max_source_mass": 0.858893632888794, "mixer_index": 3, "normalized_entropy": 0.5724589149178446, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.858893632888794, "median": 0.8705973029136658, "p05": 0.7599387168884277, "p95": 0.9270997643470764 }, { "label": "current_group_partial", "mean": 0.14110636711120605, "median": 0.12940272688865662, "p05": 0.07290027290582657, "p95": 0.2400612235069275 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6852496862411499, "group": 1, "latest_source_mass": 0.518975019454956, "layer": 3, "max_source_mass": 0.518975019454956, "mixer_index": 4, "normalized_entropy": 0.9886063241108252, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.48102498054504395, "median": 0.48102718591690063, "p05": 0.38786420226097107, "p95": 0.580143928527832 }, { "label": "current_group_partial", "mean": 0.518975019454956, "median": 0.5189728140830994, "p05": 0.41985607147216797, "p95": 0.6121357679367065 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.5890734195709229, "group": 1, "latest_source_mass": 0.3123423159122467, "layer": 3, "max_source_mass": 0.6876576542854309, "mixer_index": 5, "normalized_entropy": 0.8498533011344741, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.6876576542854309, "median": 0.6909762620925903, "p05": 0.48803940415382385, "p95": 0.8900076746940613 }, { "label": "current_group_partial", "mean": 0.3123423159122467, "median": 0.3090237081050873, "p05": 0.10999235510826111, "p95": 0.5119606256484985 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6873323321342468, "group": 1, "latest_source_mass": 0.4831485152244568, "layer": 4, "max_source_mass": 0.5168514251708984, "mixer_index": 6, "normalized_entropy": 0.9916109470127238, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.5168514251708984, "median": 0.5219584703445435, "p05": 0.4251725673675537, "p95": 0.5947866439819336 }, { "label": "current_group_partial", "mean": 0.4831485152244568, "median": 0.47804147005081177, "p05": 0.405213326215744, "p95": 0.5748274326324463 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.487301766872406, "group": 1, "latest_source_mass": 0.2060384750366211, "layer": 4, "max_source_mass": 0.7939615249633789, "mixer_index": 7, "normalized_entropy": 0.7030278424832499, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.7939615249633789, "median": 0.7953404188156128, "p05": 0.6590759754180908, "p95": 0.9329437017440796 }, { "label": "current_group_partial", "mean": 0.2060384750366211, "median": 0.20465964078903198, "p05": 0.06705628335475922, "p95": 0.3409239649772644 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.674697756767273, "group": 2, "latest_source_mass": 0.42952626943588257, "layer": 5, "max_source_mass": 0.5704737305641174, "mixer_index": 8, "normalized_entropy": 0.9733831077870527, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.5704737305641174, "median": 0.5696132779121399, "p05": 0.4695202708244324, "p95": 0.6750621199607849 }, { "label": "completed_group_1", "mean": 0.42952626943588257, "median": 0.4303867816925049, "p05": 0.3249378800392151, "p95": 0.5304797887802124 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.9691112041473389, "group": 2, "latest_source_mass": 0.2681158185005188, "layer": 5, "max_source_mass": 0.5506307482719421, "mixer_index": 9, "normalized_entropy": 0.8821230329784768, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.5506307482719421, "median": 0.543993353843689, "p05": 0.41262108087539673, "p95": 0.7161091566085815 }, { "label": "completed_group_1", "mean": 0.18125343322753906, "median": 0.1794438362121582, "p05": 0.11575750261545181, "p95": 0.25031501054763794 }, { "label": "current_group_partial", "mean": 0.2681158185005188, "median": 0.2664111256599426, "p05": 0.1564716249704361, "p95": 0.3817881643772125 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0745112895965576, "group": 2, "latest_source_mass": 0.412794291973114, "layer": 6, "max_source_mass": 0.412794291973114, "mixer_index": 10, "normalized_entropy": 0.9780623252441762, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.34262996912002563, "median": 0.3412924110889435, "p05": 0.29556700587272644, "p95": 0.39125555753707886 }, { "label": "completed_group_1", "mean": 0.24457570910453796, "median": 0.24382859468460083, "p05": 0.22120657563209534, "p95": 0.2734297513961792 }, { "label": "current_group_partial", "mean": 0.412794291973114, "median": 0.4151499271392822, "p05": 0.37735897302627563, "p95": 0.43749260902404785 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0074045658111572, "group": 2, "latest_source_mass": 0.3775783181190491, "layer": 6, "max_source_mass": 0.4399106502532959, "mixer_index": 11, "normalized_entropy": 0.9169791528842925, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.4399106502532959, "median": 0.44856905937194824, "p05": 0.22063353657722473, "p95": 0.6359403133392334 }, { "label": "completed_group_1", "mean": 0.18251106142997742, "median": 0.18562838435173035, "p05": 0.1196163073182106, "p95": 0.23045453429222107 }, { "label": "current_group_partial", "mean": 0.3775783181190491, "median": 0.3675365447998047, "p05": 0.23817461729049683, "p95": 0.5557451844215393 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.060497522354126, "group": 2, "latest_source_mass": 0.4025024175643921, "layer": 7, "max_source_mass": 0.4025024175643921, "mixer_index": 12, "normalized_entropy": 0.9653064445872968, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.3786153197288513, "median": 0.3907395303249359, "p05": 0.28039106726646423, "p95": 0.4436405599117279 }, { "label": "completed_group_1", "mean": 0.2188822478055954, "median": 0.21226292848587036, "p05": 0.1885872334241867, "p95": 0.2672030031681061 }, { "label": "current_group_partial", "mean": 0.4025024175643921, "median": 0.39815032482147217, "p05": 0.3533596992492676, "p95": 0.460100919008255 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 0.9941019415855408, "group": 2, "latest_source_mass": 0.4801652431488037, "layer": 7, "max_source_mass": 0.4801652431488037, "mixer_index": 13, "normalized_entropy": 0.9048705824970601, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.32296955585479736, "median": 0.2803739905357361, "p05": 0.1784922033548355, "p95": 0.6440245509147644 }, { "label": "completed_group_1", "mean": 0.19686520099639893, "median": 0.2053825855255127, "p05": 0.09813719242811203, "p95": 0.2504696547985077 }, { "label": "current_group_partial", "mean": 0.4801652431488037, "median": 0.5057140588760376, "p05": 0.2529003322124481, "p95": 0.598870575428009 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0503522157669067, "group": 2, "latest_source_mass": 0.4001518487930298, "layer": 8, "max_source_mass": 0.40130966901779175, "mixer_index": 14, "normalized_entropy": 0.9560717885654542, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.40130966901779175, "median": 0.396759569644928, "p05": 0.3479611873626709, "p95": 0.4733521044254303 }, { "label": "completed_group_1", "mean": 0.19853851199150085, "median": 0.19769755005836487, "p05": 0.17136667668819427, "p95": 0.22788912057876587 }, { "label": "current_group_partial", "mean": 0.4001518487930298, "median": 0.40291696786880493, "p05": 0.3506356179714203, "p95": 0.43941178917884827 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0129402875900269, "group": 2, "latest_source_mass": 0.42161279916763306, "layer": 8, "max_source_mass": 0.42161279916763306, "mixer_index": 15, "normalized_entropy": 0.9220179839951123, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.4093267023563385, "median": 0.4046899676322937, "p05": 0.2895161807537079, "p95": 0.5762131810188293 }, { "label": "completed_group_1", "mean": 0.16906049847602844, "median": 0.16888955235481262, "p05": 0.11233551055192947, "p95": 0.22477403283119202 }, { "label": "current_group_partial", "mean": 0.42161279916763306, "median": 0.4227708578109741, "p05": 0.3131049573421478, "p95": 0.519075334072113 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0774019956588745, "group": 3, "latest_source_mass": 0.3560434877872467, "layer": 9, "max_source_mass": 0.38510820269584656, "mixer_index": 16, "normalized_entropy": 0.9806935592947451, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.38510820269584656, "median": 0.3892287313938141, "p05": 0.2934730052947998, "p95": 0.4718902111053467 }, { "label": "completed_group_1", "mean": 0.25884830951690674, "median": 0.25734007358551025, "p05": 0.22217319905757904, "p95": 0.29551801085472107 }, { "label": "completed_group_2", "mean": 0.3560434877872467, "median": 0.35412120819091797, "p05": 0.29444968700408936, "p95": 0.41908568143844604 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.2740377187728882, "group": 3, "latest_source_mass": 0.3633861243724823, "layer": 9, "max_source_mass": 0.3633861243724823, "mixer_index": 17, "normalized_entropy": 0.9190239493895668, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.29550763964653015, "median": 0.30360478162765503, "p05": 0.16904757916927338, "p95": 0.45146965980529785 }, { "label": "completed_group_1", "mean": 0.09541244804859161, "median": 0.09486871212720871, "p05": 0.061360105872154236, "p95": 0.13037073612213135 }, { "label": "completed_group_2", "mean": 0.24569380283355713, "median": 0.24333015084266663, "p05": 0.1698218435049057, "p95": 0.31589236855506897 }, { "label": "current_group_partial", "mean": 0.3633861243724823, "median": 0.36472851037979126, "p05": 0.2795379161834717, "p95": 0.4399876892566681 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.360302448272705, "group": 3, "latest_source_mass": 0.28569501638412476, "layer": 10, "max_source_mass": 0.3109527826309204, "mixer_index": 18, "normalized_entropy": 0.9812507981160736, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.3109527826309204, "median": 0.32172977924346924, "p05": 0.23520725965499878, "p95": 0.37052470445632935 }, { "label": "completed_group_1", "mean": 0.20984819531440735, "median": 0.20958667993545532, "p05": 0.1891367882490158, "p95": 0.23184512555599213 }, { "label": "completed_group_2", "mean": 0.19350400567054749, "median": 0.19153150916099548, "p05": 0.16607457399368286, "p95": 0.22877082228660583 }, { "label": "current_group_partial", "mean": 0.28569501638412476, "median": 0.28060320019721985, "p05": 0.23824527859687805, "p95": 0.3439165949821472 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.2722704410552979, "group": 3, "latest_source_mass": 0.29912811517715454, "layer": 10, "max_source_mass": 0.35036465525627136, "mixer_index": 19, "normalized_entropy": 0.9177491279900463, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.35036465525627136, "median": 0.36210548877716064, "p05": 0.12807878851890564, "p95": 0.5797606706619263 }, { "label": "completed_group_1", "mean": 0.12631990015506744, "median": 0.1264108121395111, "p05": 0.06977735459804535, "p95": 0.1755298376083374 }, { "label": "completed_group_2", "mean": 0.22418735921382904, "median": 0.22563719749450684, "p05": 0.1176326721906662, "p95": 0.32001155614852905 }, { "label": "current_group_partial", "mean": 0.29912811517715454, "median": 0.28589946031570435, "p05": 0.193109393119812, "p95": 0.44119665026664734 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3558036088943481, "group": 3, "latest_source_mass": 0.314197838306427, "layer": 11, "max_source_mass": 0.314197838306427, "mixer_index": 20, "normalized_entropy": 0.978005571485618, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.298976868391037, "median": 0.2961939573287964, "p05": 0.24950163066387177, "p95": 0.35835936665534973 }, { "label": "completed_group_1", "mean": 0.1941717267036438, "median": 0.1918032318353653, "p05": 0.16809208691120148, "p95": 0.22895419597625732 }, { "label": "completed_group_2", "mean": 0.1926535815000534, "median": 0.19306811690330505, "p05": 0.16983944177627563, "p95": 0.2155935913324356 }, { "label": "current_group_partial", "mean": 0.314197838306427, "median": 0.31541913747787476, "p05": 0.2673022747039795, "p95": 0.35941022634506226 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.2696614265441895, "group": 3, "latest_source_mass": 0.2862822115421295, "layer": 11, "max_source_mass": 0.3667730689048767, "mixer_index": 21, "normalized_entropy": 0.9158671218416545, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.3667730689048767, "median": 0.4067659378051758, "p05": 0.10593366622924805, "p95": 0.5672274827957153 }, { "label": "completed_group_1", "mean": 0.15310388803482056, "median": 0.15050218999385834, "p05": 0.08651046454906464, "p95": 0.22459912300109863 }, { "label": "completed_group_2", "mean": 0.19384080171585083, "median": 0.18732509016990662, "p05": 0.1267009675502777, "p95": 0.281107634305954 }, { "label": "current_group_partial", "mean": 0.2862822115421295, "median": 0.26271265745162964, "p05": 0.17024393379688263, "p95": 0.475044310092926 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3606936931610107, "group": 3, "latest_source_mass": 0.3028184771537781, "layer": 12, "max_source_mass": 0.3028184771537781, "mixer_index": 22, "normalized_entropy": 0.9815330216461395, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.298227459192276, "median": 0.2954302132129669, "p05": 0.25287362933158875, "p95": 0.35405808687210083 }, { "label": "completed_group_1", "mean": 0.18611131608486176, "median": 0.1850002110004425, "p05": 0.15901757776737213, "p95": 0.21725836396217346 }, { "label": "completed_group_2", "mean": 0.21284276247024536, "median": 0.21402311325073242, "p05": 0.18819917738437653, "p95": 0.23418807983398438 }, { "label": "current_group_partial", "mean": 0.3028184771537781, "median": 0.3041122555732727, "p05": 0.25661808252334595, "p95": 0.34458184242248535 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.269316554069519, "group": 3, "latest_source_mass": 0.2653314769268036, "layer": 12, "max_source_mass": 0.4081852436065674, "mixer_index": 23, "normalized_entropy": 0.9156183489371815, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.4081852436065674, "median": 0.4254342317581177, "p05": 0.20765496790409088, "p95": 0.587238073348999 }, { "label": "completed_group_1", "mean": 0.14420127868652344, "median": 0.1438048928976059, "p05": 0.1064920499920845, "p95": 0.1871720254421234 }, { "label": "completed_group_2", "mean": 0.18228203058242798, "median": 0.17507661879062653, "p05": 0.12183455377817154, "p95": 0.26505547761917114 }, { "label": "current_group_partial", "mean": 0.2653314769268036, "median": 0.24924281239509583, "p05": 0.15609969198703766, "p95": 0.4061311185359955 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3636136054992676, "group": 4, "latest_source_mass": 0.3136237859725952, "layer": 13, "max_source_mass": 0.3136237859725952, "mixer_index": 24, "normalized_entropy": 0.9836392931712563, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.27427661418914795, "median": 0.27403637766838074, "p05": 0.22928935289382935, "p95": 0.319773405790329 }, { "label": "completed_group_1", "mean": 0.21341991424560547, "median": 0.20697325468063354, "p05": 0.18552586436271667, "p95": 0.26688331365585327 }, { "label": "completed_group_2", "mean": 0.19867971539497375, "median": 0.197545126080513, "p05": 0.1794913411140442, "p95": 0.222100168466568 }, { "label": "completed_group_3", "mean": 0.3136237859725952, "median": 0.31875866651535034, "p05": 0.24156072735786438, "p95": 0.3695686459541321 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.570773959159851, "group": 4, "latest_source_mass": 0.2294243425130844, "layer": 13, "max_source_mass": 0.2658740282058716, "mixer_index": 25, "normalized_entropy": 0.9759767351225285, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.2658740282058716, "median": 0.2747163474559784, "p05": 0.17927880585193634, "p95": 0.3512483835220337 }, { "label": "completed_group_1", "mean": 0.13419361412525177, "median": 0.13374894857406616, "p05": 0.11072897166013718, "p95": 0.1586270034313202 }, { "label": "completed_group_2", "mean": 0.1907704919576645, "median": 0.19074805080890656, "p05": 0.14699944853782654, "p95": 0.23316305875778198 }, { "label": "completed_group_3", "mean": 0.17973749339580536, "median": 0.17616114020347595, "p05": 0.1363743096590042, "p95": 0.23202598094940186 }, { "label": "current_group_partial", "mean": 0.2294243425130844, "median": 0.22995705902576447, "p05": 0.17975632846355438, "p95": 0.2789157032966614 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.5560038089752197, "group": 4, "latest_source_mass": 0.32468360662460327, "layer": 14, "max_source_mass": 0.32468360662460327, "mixer_index": 26, "normalized_entropy": 0.9667995248241249, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.20477107167243958, "median": 0.206403911113739, "p05": 0.16619175672531128, "p95": 0.2431035190820694 }, { "label": "completed_group_1", "mean": 0.15721669793128967, "median": 0.15658551454544067, "p05": 0.13878680765628815, "p95": 0.17778633534908295 }, { "label": "completed_group_2", "mean": 0.13811248540878296, "median": 0.13743364810943604, "p05": 0.12352626770734787, "p95": 0.15553249418735504 }, { "label": "completed_group_3", "mean": 0.1752161681652069, "median": 0.17309701442718506, "p05": 0.15223677456378937, "p95": 0.20474471151828766 }, { "label": "current_group_partial", "mean": 0.32468360662460327, "median": 0.32497870922088623, "p05": 0.2946181893348694, "p95": 0.35203880071640015 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.5659973621368408, "group": 4, "latest_source_mass": 0.26703086495399475, "layer": 14, "max_source_mass": 0.26703086495399475, "mixer_index": 27, "normalized_entropy": 0.9730088685238187, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.20531430840492249, "median": 0.20229148864746094, "p05": 0.10698921978473663, "p95": 0.3393132984638214 }, { "label": "completed_group_1", "mean": 0.15986156463623047, "median": 0.15952220559120178, "p05": 0.11969029903411865, "p95": 0.20180393755435944 }, { "label": "completed_group_2", "mean": 0.21557100117206573, "median": 0.21654027700424194, "p05": 0.1673085242509842, "p95": 0.2610440254211426 }, { "label": "completed_group_3", "mean": 0.15222224593162537, "median": 0.1510905921459198, "p05": 0.11800260841846466, "p95": 0.18893754482269287 }, { "label": "current_group_partial", "mean": 0.26703086495399475, "median": 0.258539617061615, "p05": 0.19330690801143646, "p95": 0.36521556973457336 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.5552496910095215, "group": 4, "latest_source_mass": 0.31774693727493286, "layer": 15, "max_source_mass": 0.31774693727493286, "mixer_index": 28, "normalized_entropy": 0.9663309649872576, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.2162160575389862, "median": 0.2145705670118332, "p05": 0.17727568745613098, "p95": 0.26878347992897034 }, { "label": "completed_group_1", "mean": 0.17982497811317444, "median": 0.18024154007434845, "p05": 0.15453973412513733, "p95": 0.2030555009841919 }, { "label": "completed_group_2", "mean": 0.1307673156261444, "median": 0.13107559084892273, "p05": 0.11463287472724915, "p95": 0.14630235731601715 }, { "label": "completed_group_3", "mean": 0.1554446816444397, "median": 0.15516987442970276, "p05": 0.13627474009990692, "p95": 0.1754152774810791 }, { "label": "current_group_partial", "mean": 0.31774693727493286, "median": 0.319039523601532, "p05": 0.27606821060180664, "p95": 0.35650834441185 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.5068919658660889, "group": 4, "latest_source_mass": 0.3298327624797821, "layer": 15, "max_source_mass": 0.3298327624797821, "mixer_index": 29, "normalized_entropy": 0.9362846209998111, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.20646697282791138, "median": 0.18775638937950134, "p05": 0.0806753933429718, "p95": 0.4274584949016571 }, { "label": "completed_group_1", "mean": 0.1197461187839508, "median": 0.11989086866378784, "p05": 0.0809398666024208, "p95": 0.15743419528007507 }, { "label": "completed_group_2", "mean": 0.19215428829193115, "median": 0.19620640575885773, "p05": 0.1148109883069992, "p95": 0.2491212636232376 }, { "label": "completed_group_3", "mean": 0.15179988741874695, "median": 0.15212899446487427, "p05": 0.10697861760854721, "p95": 0.19542087614536285 }, { "label": "current_group_partial", "mean": 0.3298327624797821, "median": 0.32837340235710144, "p05": 0.22746729850769043, "p95": 0.4339301884174347 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.5435223579406738, "group": 4, "latest_source_mass": 0.3224271535873413, "layer": 16, "max_source_mass": 0.3224271535873413, "mixer_index": 30, "normalized_entropy": 0.9590443632623663, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.22410975396633148, "median": 0.22760415077209473, "p05": 0.14593973755836487, "p95": 0.30093055963516235 }, { "label": "completed_group_1", "mean": 0.16986176371574402, "median": 0.16963240504264832, "p05": 0.13569067418575287, "p95": 0.20651353895664215 }, { "label": "completed_group_2", "mean": 0.13080403208732605, "median": 0.13075244426727295, "p05": 0.10945598781108856, "p95": 0.15187400579452515 }, { "label": "completed_group_3", "mean": 0.15279728174209595, "median": 0.15036612749099731, "p05": 0.12810495495796204, "p95": 0.18468116223812103 }, { "label": "current_group_partial", "mean": 0.3224271535873413, "median": 0.3164973258972168, "p05": 0.2600978910923004, "p95": 0.40530598163604736 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.4949920177459717, "group": 4, "latest_source_mass": 0.31847020983695984, "layer": 16, "max_source_mass": 0.31847020983695984, "mixer_index": 31, "normalized_entropy": 0.9288907675133354, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.27174079418182373, "median": 0.2585451304912567, "p05": 0.13639721274375916, "p95": 0.4698348045349121 }, { "label": "completed_group_1", "mean": 0.12178683280944824, "median": 0.11714950203895569, "p05": 0.08594125509262085, "p95": 0.1723237782716751 }, { "label": "completed_group_2", "mean": 0.1529826819896698, "median": 0.15223106741905212, "p05": 0.10371473431587219, "p95": 0.203575000166893 }, { "label": "completed_group_3", "mean": 0.1350194811820984, "median": 0.13363957405090332, "p05": 0.08964549005031586, "p95": 0.18468207120895386 }, { "label": "current_group_partial", "mean": 0.31847020983695984, "median": 0.3185216188430786, "p05": 0.21444711089134216, "p95": 0.4301546812057495 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.544264793395996, "group": 5, "latest_source_mass": 0.2861037850379944, "layer": 17, "max_source_mass": 0.2861037850379944, "mixer_index": 32, "normalized_entropy": 0.9595056643474137, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.24016115069389343, "median": 0.23868846893310547, "p05": 0.18435582518577576, "p95": 0.2983453571796417 }, { "label": "completed_group_1", "mean": 0.20730282366275787, "median": 0.1998077780008316, "p05": 0.16255050897598267, "p95": 0.28107208013534546 }, { "label": "completed_group_2", "mean": 0.09692573547363281, "median": 0.0967404842376709, "p05": 0.08098706603050232, "p95": 0.11221462488174438 }, { "label": "completed_group_3", "mean": 0.1695065200328827, "median": 0.1696585714817047, "p05": 0.136396586894989, "p95": 0.20130634307861328 }, { "label": "completed_group_4", "mean": 0.2861037850379944, "median": 0.28883200883865356, "p05": 0.22401469945907593, "p95": 0.3409370183944702 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.686682939529419, "group": 5, "latest_source_mass": 0.12281867861747742, "layer": 17, "max_source_mass": 0.2885604202747345, "mixer_index": 33, "normalized_entropy": 0.9413556721740635, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.16755425930023193, "median": 0.1327776312828064, "p05": 0.0790192037820816, "p95": 0.3667412996292114 }, { "label": "completed_group_1", "mean": 0.10962115228176117, "median": 0.10525335371494293, "p05": 0.07840172946453094, "p95": 0.15405339002609253 }, { "label": "completed_group_2", "mean": 0.1783454418182373, "median": 0.17896884679794312, "p05": 0.10802385956048965, "p95": 0.24048911035060883 }, { "label": "completed_group_3", "mean": 0.13310004770755768, "median": 0.13033035397529602, "p05": 0.0904831737279892, "p95": 0.17934636771678925 }, { "label": "completed_group_4", "mean": 0.2885604202747345, "median": 0.2831825315952301, "p05": 0.16967125236988068, "p95": 0.4189499616622925 }, { "label": "current_group_partial", "mean": 0.12281867861747742, "median": 0.12062528729438782, "p05": 0.07461707293987274, "p95": 0.1782219111919403 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.7070834636688232, "group": 5, "latest_source_mass": 0.2988377809524536, "layer": 18, "max_source_mass": 0.2988377809524536, "mixer_index": 34, "normalized_entropy": 0.9527414214834803, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.181177020072937, "median": 0.18419909477233887, "p05": 0.13624237477779388, "p95": 0.22192169725894928 }, { "label": "completed_group_1", "mean": 0.14557164907455444, "median": 0.144476979970932, "p05": 0.1293315887451172, "p95": 0.16486899554729462 }, { "label": "completed_group_2", "mean": 0.08318156749010086, "median": 0.0820261612534523, "p05": 0.07088497281074524, "p95": 0.0990293100476265 }, { "label": "completed_group_3", "mean": 0.11424209177494049, "median": 0.11267061531543732, "p05": 0.0979941189289093, "p95": 0.1353992223739624 }, { "label": "completed_group_4", "mean": 0.17698991298675537, "median": 0.17597860097885132, "p05": 0.14837387204170227, "p95": 0.21050144731998444 }, { "label": "current_group_partial", "mean": 0.2988377809524536, "median": 0.2989804744720459, "p05": 0.2721370458602905, "p95": 0.32572072744369507 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.7419235706329346, "group": 5, "latest_source_mass": 0.26016175746917725, "layer": 18, "max_source_mass": 0.26016175746917725, "mixer_index": 35, "normalized_entropy": 0.972186055410333, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1532188355922699, "median": 0.1451343297958374, "p05": 0.11514341086149216, "p95": 0.22103679180145264 }, { "label": "completed_group_1", "mean": 0.11420352011919022, "median": 0.11261284351348877, "p05": 0.0928458571434021, "p95": 0.14123988151550293 }, { "label": "completed_group_2", "mean": 0.16597424447536469, "median": 0.1655328869819641, "p05": 0.13871800899505615, "p95": 0.19611573219299316 }, { "label": "completed_group_3", "mean": 0.1250365972518921, "median": 0.12369821965694427, "p05": 0.10375828295946121, "p95": 0.14912742376327515 }, { "label": "completed_group_4", "mean": 0.18140508234500885, "median": 0.1796584278345108, "p05": 0.14511024951934814, "p95": 0.22286373376846313 }, { "label": "current_group_partial", "mean": 0.26016175746917725, "median": 0.26266711950302124, "p05": 0.1889389455318451, "p95": 0.3234381675720215 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.7127439975738525, "group": 5, "latest_source_mass": 0.2907913327217102, "layer": 19, "max_source_mass": 0.2907913327217102, "mixer_index": 36, "normalized_entropy": 0.9559006256078307, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.18731768429279327, "median": 0.19042012095451355, "p05": 0.13879702985286713, "p95": 0.23792000114917755 }, { "label": "completed_group_1", "mean": 0.16715484857559204, "median": 0.1667211651802063, "p05": 0.14574480056762695, "p95": 0.19026139378547668 }, { "label": "completed_group_2", "mean": 0.08496320247650146, "median": 0.08452782034873962, "p05": 0.07410907745361328, "p95": 0.097386933863163 }, { "label": "completed_group_3", "mean": 0.11362075805664062, "median": 0.11248823255300522, "p05": 0.0977015346288681, "p95": 0.13219426572322845 }, { "label": "completed_group_4", "mean": 0.15615218877792358, "median": 0.15540224313735962, "p05": 0.1289147436618805, "p95": 0.18598109483718872 }, { "label": "current_group_partial", "mean": 0.2907913327217102, "median": 0.2900415062904358, "p05": 0.25822532176971436, "p95": 0.32475030422210693 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.6076834201812744, "group": 5, "latest_source_mass": 0.3738824725151062, "layer": 19, "max_source_mass": 0.3738824725151062, "mixer_index": 37, "normalized_entropy": 0.8972652009334232, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.15032559633255005, "median": 0.12544110417366028, "p05": 0.06976518034934998, "p95": 0.35840943455696106 }, { "label": "completed_group_1", "mean": 0.08624927699565887, "median": 0.08602266013622284, "p05": 0.061998721212148666, "p95": 0.11138717085123062 }, { "label": "completed_group_2", "mean": 0.15736976265907288, "median": 0.15802210569381714, "p05": 0.10121746361255646, "p95": 0.20687000453472137 }, { "label": "completed_group_3", "mean": 0.08584291487932205, "median": 0.0827617198228836, "p05": 0.060561805963516235, "p95": 0.11351694166660309 }, { "label": "completed_group_4", "mean": 0.14632999897003174, "median": 0.1426895558834076, "p05": 0.10106660425662994, "p95": 0.20179018378257751 }, { "label": "current_group_partial", "mean": 0.3738824725151062, "median": 0.3799295425415039, "p05": 0.23631010949611664, "p95": 0.4884527325630188 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.709290862083435, "group": 5, "latest_source_mass": 0.2901708781719208, "layer": 20, "max_source_mass": 0.2901708781719208, "mixer_index": 38, "normalized_entropy": 0.9539733939957076, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.18991519510746002, "median": 0.1924753338098526, "p05": 0.14201581478118896, "p95": 0.23709386587142944 }, { "label": "completed_group_1", "mean": 0.17091524600982666, "median": 0.17006167769432068, "p05": 0.15297070145606995, "p95": 0.19153237342834473 }, { "label": "completed_group_2", "mean": 0.07822999358177185, "median": 0.07778562605381012, "p05": 0.06707640737295151, "p95": 0.09089364111423492 }, { "label": "completed_group_3", "mean": 0.116765096783638, "median": 0.11516137421131134, "p05": 0.10162825882434845, "p95": 0.13653048872947693 }, { "label": "completed_group_4", "mean": 0.1540035903453827, "median": 0.15287795662879944, "p05": 0.12593276798725128, "p95": 0.18590238690376282 }, { "label": "current_group_partial", "mean": 0.2901708781719208, "median": 0.29104921221733093, "p05": 0.25643545389175415, "p95": 0.3205650746822357 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.6690887212753296, "group": 5, "latest_source_mass": 0.3342471718788147, "layer": 20, "max_source_mass": 0.3342471718788147, "mixer_index": 39, "normalized_entropy": 0.9315361520005944, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.19029226899147034, "median": 0.19644704461097717, "p05": 0.09456793963909149, "p95": 0.30468082427978516 }, { "label": "completed_group_1", "mean": 0.1100241169333458, "median": 0.10808776319026947, "p05": 0.09035106748342514, "p95": 0.1359245330095291 }, { "label": "completed_group_2", "mean": 0.11471512168645859, "median": 0.11494918912649155, "p05": 0.08799737691879272, "p95": 0.14024731516838074 }, { "label": "completed_group_3", "mean": 0.09999909996986389, "median": 0.09682698547840118, "p05": 0.07921088486909866, "p95": 0.1264391839504242 }, { "label": "completed_group_4", "mean": 0.1507222205400467, "median": 0.1476232260465622, "p05": 0.11397356539964676, "p95": 0.19991227984428406 }, { "label": "current_group_partial", "mean": 0.3342471718788147, "median": 0.33802270889282227, "p05": 0.2548266053199768, "p95": 0.40130624175071716 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.7458693981170654, "group": 6, "latest_source_mass": 0.2361394464969635, "layer": 21, "max_source_mass": 0.2361394464969635, "mixer_index": 40, "normalized_entropy": 0.9743882636597644, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.20528709888458252, "median": 0.2140858769416809, "p05": 0.13168928027153015, "p95": 0.27098023891448975 }, { "label": "completed_group_1", "mean": 0.1722964197397232, "median": 0.17238952219486237, "p05": 0.15053534507751465, "p95": 0.19365043938159943 }, { "label": "completed_group_2", "mean": 0.10435444861650467, "median": 0.10361101478338242, "p05": 0.08888213336467743, "p95": 0.12212536484003067 }, { "label": "completed_group_3", "mean": 0.12075909972190857, "median": 0.1180175170302391, "p05": 0.10358668118715286, "p95": 0.14458929002285004 }, { "label": "completed_group_4", "mean": 0.16116347908973694, "median": 0.15841388702392578, "p05": 0.1337081342935562, "p95": 0.19791515171527863 }, { "label": "completed_group_5", "mean": 0.2361394464969635, "median": 0.23709020018577576, "p05": 0.20446214079856873, "p95": 0.26509594917297363 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.4612125158309937, "group": 6, "latest_source_mass": 0.5543455481529236, "layer": 21, "max_source_mass": 0.5543455481529236, "mixer_index": 41, "normalized_entropy": 0.7509146897354808, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.08934466540813446, "median": 0.07911523431539536, "p05": 0.04488275945186615, "p95": 0.1737232506275177 }, { "label": "completed_group_1", "mean": 0.054379627108573914, "median": 0.05344422906637192, "p05": 0.04221750423312187, "p95": 0.06924191862344742 }, { "label": "completed_group_2", "mean": 0.07567165046930313, "median": 0.07438549399375916, "p05": 0.05811355262994766, "p95": 0.09632256627082825 }, { "label": "completed_group_3", "mean": 0.05758609622716904, "median": 0.05612001568078995, "p05": 0.044792283326387405, "p95": 0.07347827404737473 }, { "label": "completed_group_4", "mean": 0.07884950935840607, "median": 0.07757881283760071, "p05": 0.05758974328637123, "p95": 0.10452556610107422 }, { "label": "completed_group_5", "mean": 0.08982288837432861, "median": 0.08912864327430725, "p05": 0.0663854256272316, "p95": 0.11604885756969452 }, { "label": "current_group_partial", "mean": 0.5543455481529236, "median": 0.5596089363098145, "p05": 0.45455610752105713, "p95": 0.6367484331130981 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.7241814136505127, "group": 6, "latest_source_mass": 0.39484071731567383, "layer": 22, "max_source_mass": 0.39484071731567383, "mixer_index": 42, "normalized_entropy": 0.886053970419732, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.13360010087490082, "median": 0.14072942733764648, "p05": 0.07699814438819885, "p95": 0.17506597936153412 }, { "label": "completed_group_1", "mean": 0.10609637200832367, "median": 0.10586297512054443, "p05": 0.08648421615362167, "p95": 0.12602579593658447 }, { "label": "completed_group_2", "mean": 0.06363911926746368, "median": 0.06324908882379532, "p05": 0.054980404675006866, "p95": 0.07316673547029495 }, { "label": "completed_group_3", "mean": 0.07426095008850098, "median": 0.07362888753414154, "p05": 0.06318790465593338, "p95": 0.08665015548467636 }, { "label": "completed_group_4", "mean": 0.09383076429367065, "median": 0.09294568747282028, "p05": 0.07638302445411682, "p95": 0.11445100605487823 }, { "label": "completed_group_5", "mean": 0.13373197615146637, "median": 0.13327741622924805, "p05": 0.11399447172880173, "p95": 0.15641780197620392 }, { "label": "current_group_partial", "mean": 0.39484071731567383, "median": 0.3879151940345764, "p05": 0.3302172124385834, "p95": 0.4852263331413269 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.3979300260543823, "group": 6, "latest_source_mass": 0.5830056071281433, "layer": 22, "max_source_mass": 0.5830056071281433, "mixer_index": 43, "normalized_entropy": 0.7183939231382495, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.08959861099720001, "median": 0.07544538378715515, "p05": 0.04165133833885193, "p95": 0.20099139213562012 }, { "label": "completed_group_1", "mean": 0.05414731800556183, "median": 0.05336771905422211, "p05": 0.04069708660244942, "p95": 0.07046990841627121 }, { "label": "completed_group_2", "mean": 0.07668881118297577, "median": 0.07607124000787735, "p05": 0.0555989071726799, "p95": 0.09980960935354233 }, { "label": "completed_group_3", "mean": 0.05603286623954773, "median": 0.054573021829128265, "p05": 0.04162490367889404, "p95": 0.0730118378996849 }, { "label": "completed_group_4", "mean": 0.06979228556156158, "median": 0.06884932518005371, "p05": 0.04576495662331581, "p95": 0.09806846082210541 }, { "label": "completed_group_5", "mean": 0.07073447108268738, "median": 0.07023533433675766, "p05": 0.04956750199198723, "p95": 0.09379896521568298 }, { "label": "current_group_partial", "mean": 0.5830056071281433, "median": 0.5862616896629333, "p05": 0.4658474922180176, "p95": 0.6849814057350159 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.7459560632705688, "group": 6, "latest_source_mass": 0.37691548466682434, "layer": 23, "max_source_mass": 0.37691548466682434, "mixer_index": 44, "normalized_entropy": 0.897243926765161, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.14318783581256866, "median": 0.1493408977985382, "p05": 0.08292607218027115, "p95": 0.19464610517024994 }, { "label": "completed_group_1", "mean": 0.1094653457403183, "median": 0.10858799517154694, "p05": 0.09432064741849899, "p95": 0.1279931515455246 }, { "label": "completed_group_2", "mean": 0.06349228322505951, "median": 0.06325754523277283, "p05": 0.05439673364162445, "p95": 0.0737837627530098 }, { "label": "completed_group_3", "mean": 0.07859592139720917, "median": 0.0775163471698761, "p05": 0.06603655964136124, "p95": 0.09357087314128876 }, { "label": "completed_group_4", "mean": 0.09567482024431229, "median": 0.09468592703342438, "p05": 0.07671141624450684, "p95": 0.11774884164333344 }, { "label": "completed_group_5", "mean": 0.13266831636428833, "median": 0.13199329376220703, "p05": 0.11244359612464905, "p95": 0.15504513680934906 }, { "label": "current_group_partial", "mean": 0.37691548466682434, "median": 0.37180638313293457, "p05": 0.31207239627838135, "p95": 0.4595233201980591 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.4055393934249878, "group": 6, "latest_source_mass": 0.5772827863693237, "layer": 23, "max_source_mass": 0.5772827863693237, "mixer_index": 45, "normalized_entropy": 0.7223043644164862, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.10154955089092255, "median": 0.08766824007034302, "p05": 0.04901552200317383, "p95": 0.20287738740444183 }, { "label": "completed_group_1", "mean": 0.05619276314973831, "median": 0.055477917194366455, "p05": 0.041608311235904694, "p95": 0.07371112704277039 }, { "label": "completed_group_2", "mean": 0.07791627198457718, "median": 0.07736755907535553, "p05": 0.055920060724020004, "p95": 0.10158214718103409 }, { "label": "completed_group_3", "mean": 0.053915880620479584, "median": 0.05215863138437271, "p05": 0.03798118978738785, "p95": 0.07156119495630264 }, { "label": "completed_group_4", "mean": 0.06398355215787888, "median": 0.06229078769683838, "p05": 0.040616974234580994, "p95": 0.09270931780338287 }, { "label": "completed_group_5", "mean": 0.06915920227766037, "median": 0.0684593990445137, "p05": 0.046986356377601624, "p95": 0.09408729523420334 }, { "label": "current_group_partial", "mean": 0.5772827863693237, "median": 0.5824689269065857, "p05": 0.4518115520477295, "p95": 0.6926202178001404 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.7853620052337646, "group": 6, "latest_source_mass": 0.3407441973686218, "layer": 24, "max_source_mass": 0.3407441973686218, "mixer_index": 46, "normalized_entropy": 0.9174945750195659, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.15284113585948944, "median": 0.1593906283378601, "p05": 0.08678612112998962, "p95": 0.211342453956604 }, { "label": "completed_group_1", "mean": 0.12206703424453735, "median": 0.12153112143278122, "p05": 0.1060115247964859, "p95": 0.1399766057729721 }, { "label": "completed_group_2", "mean": 0.06508444249629974, "median": 0.06482555717229843, "p05": 0.055350616574287415, "p95": 0.07557765394449234 }, { "label": "completed_group_3", "mean": 0.08347313106060028, "median": 0.08190013468265533, "p05": 0.06966532766819, "p95": 0.10050469636917114 }, { "label": "completed_group_4", "mean": 0.09811970591545105, "median": 0.09664031863212585, "p05": 0.07688035070896149, "p95": 0.12369741499423981 }, { "label": "completed_group_5", "mean": 0.13767032325267792, "median": 0.1373325139284134, "p05": 0.11642646044492722, "p95": 0.1606694608926773 }, { "label": "current_group_partial", "mean": 0.3407441973686218, "median": 0.33805108070373535, "p05": 0.271301805973053, "p95": 0.41781002283096313 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.509842872619629, "group": 6, "latest_source_mass": 0.5227814316749573, "layer": 24, "max_source_mass": 0.5227814316749573, "mixer_index": 47, "normalized_entropy": 0.7759057494780099, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.11788781732320786, "median": 0.10097194463014603, "p05": 0.06265684217214584, "p95": 0.2408325970172882 }, { "label": "completed_group_1", "mean": 0.06822571158409119, "median": 0.0673636943101883, "p05": 0.05209728330373764, "p95": 0.08648915588855743 }, { "label": "completed_group_2", "mean": 0.08556114137172699, "median": 0.08463940769433975, "p05": 0.06234566867351532, "p95": 0.1125786080956459 }, { "label": "completed_group_3", "mean": 0.06067090481519699, "median": 0.058974385261535645, "p05": 0.043267179280519485, "p95": 0.07997901737689972 }, { "label": "completed_group_4", "mean": 0.07228855788707733, "median": 0.06972275674343109, "p05": 0.04689513146877289, "p95": 0.106512151658535 }, { "label": "completed_group_5", "mean": 0.07258446514606476, "median": 0.07082539796829224, "p05": 0.050949547439813614, "p95": 0.09970614314079285 }, { "label": "current_group_partial", "mean": 0.5227814316749573, "median": 0.5290273427963257, "p05": 0.38773614168167114, "p95": 0.6392589807510376 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.839367389678955, "group": 7, "latest_source_mass": 0.29583215713500977, "layer": 25, "max_source_mass": 0.29583215713500977, "mixer_index": 48, "normalized_entropy": 0.9452478525649903, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.15906387567520142, "median": 0.15846745669841766, "p05": 0.09519976377487183, "p95": 0.24517810344696045 }, { "label": "completed_group_1", "mean": 0.13139380514621735, "median": 0.1308426856994629, "p05": 0.11478274315595627, "p95": 0.15000829100608826 }, { "label": "completed_group_2", "mean": 0.07717745751142502, "median": 0.07715500891208649, "p05": 0.06616044044494629, "p95": 0.08791667222976685 }, { "label": "completed_group_3", "mean": 0.09607303142547607, "median": 0.09436552226543427, "p05": 0.08152313530445099, "p95": 0.11384639143943787 }, { "label": "completed_group_4", "mean": 0.1107850894331932, "median": 0.1089654266834259, "p05": 0.08814431726932526, "p95": 0.139452263712883 }, { "label": "completed_group_5", "mean": 0.12967455387115479, "median": 0.1299237608909607, "p05": 0.10832332074642181, "p95": 0.15138982236385345 }, { "label": "completed_group_6", "mean": 0.29583215713500977, "median": 0.29647016525268555, "p05": 0.2276897132396698, "p95": 0.3604946434497833 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.5841834545135498, "group": 7, "latest_source_mass": 0.40767741203308105, "layer": 25, "max_source_mass": 0.40767741203308105, "mixer_index": 49, "normalized_entropy": 0.7618312045616817, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.07896879315376282, "median": 0.06809917092323303, "p05": 0.025932887569069862, "p95": 0.19368338584899902 }, { "label": "completed_group_1", "mean": 0.048752427101135254, "median": 0.04767637699842453, "p05": 0.034443892538547516, "p95": 0.0667153000831604 }, { "label": "completed_group_2", "mean": 0.055707164108753204, "median": 0.0549345538020134, "p05": 0.038696639239788055, "p95": 0.0750277042388916 }, { "label": "completed_group_3", "mean": 0.04139826446771622, "median": 0.03878612071275711, "p05": 0.026673054322600365, "p95": 0.05746251344680786 }, { "label": "completed_group_4", "mean": 0.04449018836021423, "median": 0.04196460545063019, "p05": 0.026162121444940567, "p95": 0.07164917886257172 }, { "label": "completed_group_5", "mean": 0.04285799711942673, "median": 0.04109475016593933, "p05": 0.029393533244729042, "p95": 0.06277710199356079 }, { "label": "completed_group_6", "mean": 0.2801477909088135, "median": 0.2763082981109619, "p05": 0.1455708146095276, "p95": 0.4254714846611023 }, { "label": "current_group_partial", "mean": 0.40767741203308105, "median": 0.4120897948741913, "p05": 0.21399420499801636, "p95": 0.5853031277656555 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.8125314712524414, "group": 7, "latest_source_mass": 0.3700707256793976, "layer": 26, "max_source_mass": 0.3700707256793976, "mixer_index": 50, "normalized_entropy": 0.8716433883436914, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.10889513790607452, "median": 0.11333495378494263, "p05": 0.04850132763385773, "p95": 0.1620434820652008 }, { "label": "completed_group_1", "mean": 0.08656036853790283, "median": 0.08640051633119583, "p05": 0.0650845319032669, "p95": 0.1084962785243988 }, { "label": "completed_group_2", "mean": 0.048123255372047424, "median": 0.04800213500857353, "p05": 0.03576180338859558, "p95": 0.06107594817876816 }, { "label": "completed_group_3", "mean": 0.06561170518398285, "median": 0.06379580497741699, "p05": 0.050968848168849945, "p95": 0.08332888036966324 }, { "label": "completed_group_4", "mean": 0.06917259097099304, "median": 0.06724178791046143, "p05": 0.050207968801259995, "p95": 0.09373342245817184 }, { "label": "completed_group_5", "mean": 0.08286689221858978, "median": 0.08183546364307404, "p05": 0.06535022705793381, "p95": 0.10340933501720428 }, { "label": "completed_group_6", "mean": 0.16869932413101196, "median": 0.16509854793548584, "p05": 0.11059704422950745, "p95": 0.23654375970363617 }, { "label": "current_group_partial", "mean": 0.3700707256793976, "median": 0.3694133758544922, "p05": 0.23441238701343536, "p95": 0.5025027990341187 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.50089693069458, "group": 7, "latest_source_mass": 0.49096518754959106, "layer": 26, "max_source_mass": 0.49096518754959106, "mixer_index": 51, "normalized_entropy": 0.7217788529328457, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.07974998652935028, "median": 0.07402504235506058, "p05": 0.04579866677522659, "p95": 0.1359037309885025 }, { "label": "completed_group_1", "mean": 0.04166398197412491, "median": 0.04038450866937637, "p05": 0.027992289513349533, "p95": 0.05969971418380737 }, { "label": "completed_group_2", "mean": 0.04607272148132324, "median": 0.044438350945711136, "p05": 0.030038591474294662, "p95": 0.06808309257030487 }, { "label": "completed_group_3", "mean": 0.03586764633655548, "median": 0.03317485377192497, "p05": 0.02203454077243805, "p95": 0.05572832003235817 }, { "label": "completed_group_4", "mean": 0.04223340004682541, "median": 0.03930673003196716, "p05": 0.02465122565627098, "p95": 0.06968094408512115 }, { "label": "completed_group_5", "mean": 0.04095568507909775, "median": 0.03919431194663048, "p05": 0.026239857077598572, "p95": 0.06058088690042496 }, { "label": "completed_group_6", "mean": 0.22249138355255127, "median": 0.2125401496887207, "p05": 0.10647299885749817, "p95": 0.3711881935596466 }, { "label": "current_group_partial", "mean": 0.49096518754959106, "median": 0.4988659620285034, "p05": 0.2878011167049408, "p95": 0.6676807403564453 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.8059929609298706, "group": 7, "latest_source_mass": 0.37139976024627686, "layer": 27, "max_source_mass": 0.37139976024627686, "mixer_index": 52, "normalized_entropy": 0.8684990295379666, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.10711923241615295, "median": 0.10972198843955994, "p05": 0.04896187782287598, "p95": 0.16201043128967285 }, { "label": "completed_group_1", "mean": 0.07651707530021667, "median": 0.07666032761335373, "p05": 0.05141128972172737, "p95": 0.10152198374271393 }, { "label": "completed_group_2", "mean": 0.058284737169742584, "median": 0.05812416970729828, "p05": 0.039273619651794434, "p95": 0.07745087146759033 }, { "label": "completed_group_3", "mean": 0.06609587371349335, "median": 0.06383530795574188, "p05": 0.04481930285692215, "p95": 0.09336265176534653 }, { "label": "completed_group_4", "mean": 0.07139429450035095, "median": 0.0694514587521553, "p05": 0.0485839769244194, "p95": 0.10034126043319702 }, { "label": "completed_group_5", "mean": 0.07493151724338531, "median": 0.07319007813930511, "p05": 0.051876723766326904, "p95": 0.10402432829141617 }, { "label": "completed_group_6", "mean": 0.17425751686096191, "median": 0.16937679052352905, "p05": 0.10449051856994629, "p95": 0.2624811828136444 }, { "label": "current_group_partial", "mean": 0.37139976024627686, "median": 0.3732905089855194, "p05": 0.23830299079418182, "p95": 0.5014542937278748 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.5622732639312744, "group": 7, "latest_source_mass": 0.4768889844417572, "layer": 27, "max_source_mass": 0.4768889844417572, "mixer_index": 53, "normalized_entropy": 0.7512946301290215, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.08775827288627625, "median": 0.08504067361354828, "p05": 0.0428270623087883, "p95": 0.144285649061203 }, { "label": "completed_group_1", "mean": 0.047765426337718964, "median": 0.04646281152963638, "p05": 0.031814731657505035, "p95": 0.06807165592908859 }, { "label": "completed_group_2", "mean": 0.053928617388010025, "median": 0.052262187004089355, "p05": 0.035437777638435364, "p95": 0.0784272700548172 }, { "label": "completed_group_3", "mean": 0.04293380305171013, "median": 0.03996016085147858, "p05": 0.02556057646870613, "p95": 0.06623870134353638 }, { "label": "completed_group_4", "mean": 0.04174989461898804, "median": 0.03903091698884964, "p05": 0.025013400241732597, "p95": 0.06869364529848099 }, { "label": "completed_group_5", "mean": 0.04734005779027939, "median": 0.04609068110585213, "p05": 0.029884062707424164, "p95": 0.06911227852106094 }, { "label": "completed_group_6", "mean": 0.20163494348526, "median": 0.19222964346408844, "p05": 0.09963390231132507, "p95": 0.3307305574417114 }, { "label": "current_group_partial", "mean": 0.4768889844417572, "median": 0.48383551836013794, "p05": 0.276212215423584, "p95": 0.6487575173377991 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.8757455348968506, "group": 7, "latest_source_mass": 0.3109493851661682, "layer": 28, "max_source_mass": 0.3109493851661682, "mixer_index": 54, "normalized_entropy": 0.9020429270551009, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.11563669890165329, "median": 0.12087491154670715, "p05": 0.05733206868171692, "p95": 0.1688953936100006 }, { "label": "completed_group_1", "mean": 0.08108697831630707, "median": 0.08185897022485733, "p05": 0.05175650864839554, "p95": 0.10753051191568375 }, { "label": "completed_group_2", "mean": 0.07280858606100082, "median": 0.07357072830200195, "p05": 0.04870978742837906, "p95": 0.0948689877986908 }, { "label": "completed_group_3", "mean": 0.07553227245807648, "median": 0.07362619787454605, "p05": 0.056317005306482315, "p95": 0.09756095707416534 }, { "label": "completed_group_4", "mean": 0.07606014609336853, "median": 0.07456053793430328, "p05": 0.05528895929455757, "p95": 0.1022825762629509 }, { "label": "completed_group_5", "mean": 0.06935927271842957, "median": 0.06843917071819305, "p05": 0.04978285729885101, "p95": 0.09167985618114471 }, { "label": "completed_group_6", "mean": 0.19856664538383484, "median": 0.19508783519268036, "p05": 0.1281973123550415, "p95": 0.27796900272369385 }, { "label": "current_group_partial", "mean": 0.3109493851661682, "median": 0.3105033040046692, "p05": 0.18172834813594818, "p95": 0.43916669487953186 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.6924852132797241, "group": 7, "latest_source_mass": 0.3315581679344177, "layer": 28, "max_source_mass": 0.3315581679344177, "mixer_index": 55, "normalized_entropy": 0.8139133413255193, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.08638577908277512, "median": 0.058608733117580414, "p05": 0.026544082909822464, "p95": 0.2740764319896698 }, { "label": "completed_group_1", "mean": 0.059912845492362976, "median": 0.05901291221380234, "p05": 0.03629481792449951, "p95": 0.08707985281944275 }, { "label": "completed_group_2", "mean": 0.06609142571687698, "median": 0.0647939145565033, "p05": 0.038691695779561996, "p95": 0.09703366458415985 }, { "label": "completed_group_3", "mean": 0.056745611131191254, "median": 0.05418796092271805, "p05": 0.032174739986658096, "p95": 0.08370660990476608 }, { "label": "completed_group_4", "mean": 0.06277773529291153, "median": 0.0600711889564991, "p05": 0.03501531109213829, "p95": 0.10094858705997467 }, { "label": "completed_group_5", "mean": 0.05352145433425903, "median": 0.05179818347096443, "p05": 0.032271772623062134, "p95": 0.08084158599376678 }, { "label": "completed_group_6", "mean": 0.2830069661140442, "median": 0.2759752869606018, "p05": 0.12981083989143372, "p95": 0.46204638481140137 }, { "label": "current_group_partial", "mean": 0.3315581679344177, "median": 0.3204721510410309, "p05": 0.13446375727653503, "p95": 0.5667697787284851 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.8980522155761719, "group": 8, "latest_source_mass": 0.20123262703418732, "layer": 29, "max_source_mass": 0.26602888107299805, "mixer_index": 56, "normalized_entropy": 0.9127701729200177, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.11603894829750061, "median": 0.09991352260112762, "p05": 0.030035395175218582, "p95": 0.2941743731498718 }, { "label": "completed_group_1", "mean": 0.08804173022508621, "median": 0.08880250155925751, "p05": 0.05664174258708954, "p95": 0.11779069900512695 }, { "label": "completed_group_2", "mean": 0.07975024729967117, "median": 0.07984748482704163, "p05": 0.05428915470838547, "p95": 0.10499341040849686 }, { "label": "completed_group_3", "mean": 0.09027000516653061, "median": 0.08898856490850449, "p05": 0.06189651042222977, "p95": 0.11996553838253021 }, { "label": "completed_group_4", "mean": 0.08430293202400208, "median": 0.0819794088602066, "p05": 0.05464346706867218, "p95": 0.12159879505634308 }, { "label": "completed_group_5", "mean": 0.07433462142944336, "median": 0.07287181913852692, "p05": 0.052520088851451874, "p95": 0.1010315865278244 }, { "label": "completed_group_6", "mean": 0.26602888107299805, "median": 0.2639757990837097, "p05": 0.15474572777748108, "p95": 0.3846370279788971 }, { "label": "completed_group_7", "mean": 0.20123262703418732, "median": 0.18140269815921783, "p05": 0.08419936150312424, "p95": 0.3782624900341034 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.7512003183364868, "group": 8, "latest_source_mass": 0.33482813835144043, "layer": 29, "max_source_mass": 0.33482813835144043, "mixer_index": 57, "normalized_entropy": 0.7970056117156376, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.07744461297988892, "median": 0.06970229744911194, "p05": 0.014401955530047417, "p95": 0.17144526541233063 }, { "label": "completed_group_1", "mean": 0.046086348593235016, "median": 0.04394831880927086, "p05": 0.028066720813512802, "p95": 0.0718703418970108 }, { "label": "completed_group_2", "mean": 0.05025433003902435, "median": 0.04861236736178398, "p05": 0.029644077643752098, "p95": 0.07761462777853012 }, { "label": "completed_group_3", "mean": 0.03748488053679466, "median": 0.03434944897890091, "p05": 0.020774798467755318, "p95": 0.05938909202814102 }, { "label": "completed_group_4", "mean": 0.04005061462521553, "median": 0.037498839199543, "p05": 0.019541850313544273, "p95": 0.06922487914562225 }, { "label": "completed_group_5", "mean": 0.041623204946517944, "median": 0.039552345871925354, "p05": 0.022081367671489716, "p95": 0.06821337342262268 }, { "label": "completed_group_6", "mean": 0.198423832654953, "median": 0.18318486213684082, "p05": 0.07548617571592331, "p95": 0.3721975088119507 }, { "label": "completed_group_7", "mean": 0.17380405962467194, "median": 0.1514919549226761, "p05": 0.05777408182621002, "p95": 0.36326974630355835 }, { "label": "current_group_partial", "mean": 0.33482813835144043, "median": 0.33602219820022583, "p05": 0.0795319601893425, "p95": 0.5979493856430054 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.0160694122314453, "group": 8, "latest_source_mass": 0.21711516380310059, "layer": 30, "max_source_mass": 0.21711516380310059, "mixer_index": 58, "normalized_entropy": 0.9175527313077866, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.09385271370410919, "median": 0.09211117774248123, "p05": 0.032377734780311584, "p95": 0.16496515274047852 }, { "label": "completed_group_1", "mean": 0.06208505108952522, "median": 0.06140740588307381, "p05": 0.0407925546169281, "p95": 0.0863042101264 }, { "label": "completed_group_2", "mean": 0.06560768187046051, "median": 0.06516212224960327, "p05": 0.041897669434547424, "p95": 0.09085705131292343 }, { "label": "completed_group_3", "mean": 0.07029052078723907, "median": 0.06780634075403214, "p05": 0.04846132919192314, "p95": 0.09917552024126053 }, { "label": "completed_group_4", "mean": 0.06935097277164459, "median": 0.06708167493343353, "p05": 0.04668639227747917, "p95": 0.09815650433301926 }, { "label": "completed_group_5", "mean": 0.056832700967788696, "median": 0.05595892667770386, "p05": 0.038822539150714874, "p95": 0.0771557092666626 }, { "label": "completed_group_6", "mean": 0.19425815343856812, "median": 0.19145764410495758, "p05": 0.12249615788459778, "p95": 0.27496030926704407 }, { "label": "completed_group_7", "mean": 0.17060703039169312, "median": 0.16317546367645264, "p05": 0.09239593893289566, "p95": 0.2726203501224518 }, { "label": "current_group_partial", "mean": 0.21711516380310059, "median": 0.21198657155036926, "p05": 0.11019522696733475, "p95": 0.3391069769859314 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 1.7837426662445068, "group": 8, "latest_source_mass": 0.29026859998703003, "layer": 30, "max_source_mass": 0.29026859998703003, "mixer_index": 59, "normalized_entropy": 0.8118162725118464, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.08836327493190765, "median": 0.07904411852359772, "p05": 0.02631576545536518, "p95": 0.17534467577934265 }, { "label": "completed_group_1", "mean": 0.04953558370471001, "median": 0.04658038914203644, "p05": 0.025181813165545464, "p95": 0.08318150043487549 }, { "label": "completed_group_2", "mean": 0.04813014343380928, "median": 0.045740917325019836, "p05": 0.026714935898780823, "p95": 0.07843323051929474 }, { "label": "completed_group_3", "mean": 0.0373263955116272, "median": 0.03401809185743332, "p05": 0.01990434154868126, "p95": 0.06290259957313538 }, { "label": "completed_group_4", "mean": 0.037015400826931, "median": 0.03405017405748367, "p05": 0.018646355718374252, "p95": 0.06544196605682373 }, { "label": "completed_group_5", "mean": 0.03390160948038101, "median": 0.03198609501123428, "p05": 0.018189728260040283, "p95": 0.05570102855563164 }, { "label": "completed_group_6", "mean": 0.1840541511774063, "median": 0.17327937483787537, "p05": 0.07966600358486176, "p95": 0.32982927560806274 }, { "label": "completed_group_7", "mean": 0.23140482604503632, "median": 0.22406929731369019, "p05": 0.08831030130386353, "p95": 0.40350067615509033 }, { "label": "current_group_partial", "mean": 0.29026859998703003, "median": 0.2745037376880646, "p05": 0.10280418395996094, "p95": 0.5399085879325867 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 1.9973840713500977, "group": 8, "latest_source_mass": 0.19101998209953308, "layer": 31, "max_source_mass": 0.20831164717674255, "mixer_index": 60, "normalized_entropy": 0.9090486661912383, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.10261696577072144, "median": 0.10389091819524765, "p05": 0.01371697522699833, "p95": 0.21163305640220642 }, { "label": "completed_group_1", "mean": 0.06039813905954361, "median": 0.06085003539919853, "p05": 0.026107093319296837, "p95": 0.09348798543214798 }, { "label": "completed_group_2", "mean": 0.06671924144029617, "median": 0.06656201928853989, "p05": 0.034684643149375916, "p95": 0.09819839894771576 }, { "label": "completed_group_3", "mean": 0.08377155661582947, "median": 0.0804532840847969, "p05": 0.05448959022760391, "p95": 0.12039553374052048 }, { "label": "completed_group_4", "mean": 0.07002593576908112, "median": 0.06737451255321503, "p05": 0.044841229915618896, "p95": 0.10655929893255234 }, { "label": "completed_group_5", "mean": 0.05768897384405136, "median": 0.056077782064676285, "p05": 0.039495617151260376, "p95": 0.08242568373680115 }, { "label": "completed_group_6", "mean": 0.20831164717674255, "median": 0.2021285891532898, "p05": 0.11652229726314545, "p95": 0.31706833839416504 }, { "label": "completed_group_7", "mean": 0.1594475358724594, "median": 0.1546279788017273, "p05": 0.06593266129493713, "p95": 0.2735573351383209 }, { "label": "current_group_partial", "mean": 0.19101998209953308, "median": 0.1713225245475769, "p05": 0.06408411264419556, "p95": 0.3850628137588501 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 1.969987154006958, "group": 8, "latest_source_mass": 0.16864418983459473, "layer": 31, "max_source_mass": 0.24503237009048462, "mixer_index": 61, "normalized_entropy": 0.8965797917640489, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.08120635151863098, "median": 0.061164144426584244, "p05": 0.0259944386780262, "p95": 0.23622018098831177 }, { "label": "completed_group_1", "mean": 0.07894632965326309, "median": 0.07658770680427551, "p05": 0.0496862456202507, "p95": 0.11596276611089706 }, { "label": "completed_group_2", "mean": 0.08615730702877045, "median": 0.08467075228691101, "p05": 0.050409119576215744, "p95": 0.12811316549777985 }, { "label": "completed_group_3", "mean": 0.07609779387712479, "median": 0.0722871869802475, "p05": 0.04099677503108978, "p95": 0.11950768530368805 }, { "label": "completed_group_4", "mean": 0.058728985488414764, "median": 0.056080836802721024, "p05": 0.03323862701654434, "p95": 0.09344407916069031 }, { "label": "completed_group_5", "mean": 0.05076971277594566, "median": 0.04946017265319824, "p05": 0.03047419898211956, "p95": 0.07563400268554688 }, { "label": "completed_group_6", "mean": 0.24503237009048462, "median": 0.24110376834869385, "p05": 0.10613486170768738, "p95": 0.3991503119468689 }, { "label": "completed_group_7", "mean": 0.1544169783592224, "median": 0.1400495171546936, "p05": 0.06199025735259056, "p95": 0.2995939254760742 }, { "label": "current_group_partial", "mean": 0.16864418983459473, "median": 0.15222924947738647, "p05": 0.0543956533074379, "p95": 0.3329208791255951 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.071648359298706, "group": 8, "latest_source_mass": 0.11520501971244812, "layer": 32, "max_source_mass": 0.20955611765384674, "mixer_index": 62, "normalized_entropy": 0.9428478002054053, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.10569609701633453, "median": 0.08689188957214355, "p05": 0.03173588588833809, "p95": 0.24571751058101654 }, { "label": "completed_group_1", "mean": 0.09922641515731812, "median": 0.09795533865690231, "p05": 0.06407676637172699, "p95": 0.14120742678642273 }, { "label": "completed_group_2", "mean": 0.08180510997772217, "median": 0.080217644572258, "p05": 0.054607540369033813, "p95": 0.11442367732524872 }, { "label": "completed_group_3", "mean": 0.10646799206733704, "median": 0.10510502755641937, "p05": 0.07415887713432312, "p95": 0.14216329157352448 }, { "label": "completed_group_4", "mean": 0.0826798751950264, "median": 0.08022409677505493, "p05": 0.05491747707128525, "p95": 0.11860857903957367 }, { "label": "completed_group_5", "mean": 0.06615597754716873, "median": 0.06485182791948318, "p05": 0.04477996751666069, "p95": 0.09238749742507935 }, { "label": "completed_group_6", "mean": 0.20955611765384674, "median": 0.206287682056427, "p05": 0.129981130361557, "p95": 0.2996031939983368 }, { "label": "completed_group_7", "mean": 0.13320739567279816, "median": 0.12465103715658188, "p05": 0.06303662061691284, "p95": 0.23065781593322754 }, { "label": "current_group_partial", "mean": 0.11520501971244812, "median": 0.10386130958795547, "p05": 0.03912060335278511, "p95": 0.22735844552516937 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 1.9781701564788818, "group": 8, "latest_source_mass": 0.16112206876277924, "layer": 32, "max_source_mass": 0.22251944243907928, "mixer_index": 63, "normalized_entropy": 0.9003040366848136, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.09446848928928375, "median": 0.074150450527668, "p05": 0.042592622339725494, "p95": 0.19443637132644653 }, { "label": "completed_group_1", "mean": 0.07056106626987457, "median": 0.06868506222963333, "p05": 0.04818308353424072, "p95": 0.0982733890414238 }, { "label": "completed_group_2", "mean": 0.08033448457717896, "median": 0.08047192543745041, "p05": 0.048296794295310974, "p95": 0.11253741383552551 }, { "label": "completed_group_3", "mean": 0.0658978745341301, "median": 0.06256154924631119, "p05": 0.04223886504769325, "p95": 0.0967884361743927 }, { "label": "completed_group_4", "mean": 0.058234501630067825, "median": 0.0563921295106411, "p05": 0.03529864549636841, "p95": 0.08846980333328247 }, { "label": "completed_group_5", "mean": 0.04316519945859909, "median": 0.042268507182598114, "p05": 0.029443470761179924, "p95": 0.05996086448431015 }, { "label": "completed_group_6", "mean": 0.2036968618631363, "median": 0.19645483791828156, "p05": 0.09398645162582397, "p95": 0.3395216464996338 }, { "label": "completed_group_7", "mean": 0.22251944243907928, "median": 0.2130982130765915, "p05": 0.11867167800664902, "p95": 0.35465875267982483 }, { "label": "current_group_partial", "mean": 0.16112206876277924, "median": 0.14023703336715698, "p05": 0.06008770316839218, "p95": 0.3247341811656952 } ], "sources": 9 }, { "branch": "output", "entropy_mean": 1.6036605834960938, "group": null, "latest_source_mass": 0.31639498472213745, "layer": null, "max_source_mass": 0.3412485122680664, "mixer_index": null, "normalized_entropy": 0.7298573846467136, "offset": null, "source_summaries": [ { "label": "embedding", "mean": 0.09759747982025146, "median": 0.09915056824684143, "p05": 0.025461740791797638, "p95": 0.16398483514785767 }, { "label": "completed_group_1", "mean": 0.04215940088033676, "median": 0.04050712287425995, "p05": 0.025720715522766113, "p95": 0.06398092210292816 }, { "label": "completed_group_2", "mean": 0.041145481169223785, "median": 0.03881259262561798, "p05": 0.023510465398430824, "p95": 0.0662468671798706 }, { "label": "completed_group_3", "mean": 0.029554028064012527, "median": 0.027420252561569214, "p05": 0.01627083122730255, "p95": 0.04924973472952843 }, { "label": "completed_group_4", "mean": 0.0289996936917305, "median": 0.027226522564888, "p05": 0.01560397818684578, "p95": 0.048351090401411057 }, { "label": "completed_group_5", "mean": 0.03647579252719879, "median": 0.0347352996468544, "p05": 0.020744025707244873, "p95": 0.058257561177015305 }, { "label": "completed_group_6", "mean": 0.06642461568117142, "median": 0.057809896767139435, "p05": 0.022837381809949875, "p95": 0.14078694581985474 }, { "label": "completed_group_7", "mean": 0.3412485122680664, "median": 0.3336990475654602, "p05": 0.0819513276219368, "p95": 0.6423859596252441 }, { "label": "completed_group_8", "mean": 0.31639498472213745, "median": 0.30713334679603577, "p05": 0.05841828137636185, "p95": 0.6156357526779175 } ], "sources": 9 } ], "mode": "uniform_group_6_only", "optimizer_state_after": "ed03e6fbd4a12d8b063dcb22e0437754285f54d585374cd52fbd534f05d24637", "optimizer_state_before": "ed03e6fbd4a12d8b063dcb22e0437754285f54d585374cd52fbd534f05d24637", "optimizer_state_unchanged": true, "positions": { "attention_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "5e6ecbcd78543fc42d4098a51169dca446d404ce384015643e84d2c78433c949", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.0933690071105957, 0.24537226557731628, 0.07698935270309448, 0.0523640513420105, 0.09380833059549332, 0.03647654876112938, 0.039120711386203766, 0.06371714919805527, 0.05156813934445381, 0.03728010877966881, 0.03983496502041817, 0.040284499526023865, 0.046871740370988846, 0.04314620420336723, 0.04033593460917473, 0.04471700266003609, 0.08397205173969269, 0.0528092160820961, 0.04443390294909477, 0.07560808956623077, 0.03708760067820549, 0.03599191829562187, 0.033304885029792786, 0.04061548039317131, 0.025487789884209633, 0.021565141156315804, 0.02342291921377182, 0.017244853079319, 0.024166326969861984, 0.016819356009364128, 0.021590016782283783, 0.026201345026493073 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.840096479099884e-05, "first_to_last_ratio": 0.7650907658930362, "imbalance_abs_log_ratio": 0.26776080396755425, "last_quartile_mean": 2.4050695174082648e-05, "mean": 1.9518664629458726e-05, "normalized": [ 0.8715907514433461, 1.0093392336431617, 0.9039238226621499, 0.8406061729195592, 1.0862490746822901, 1.016450645465141, 0.9600711630441814, 0.8536640927150235, 0.8217077979969767, 0.7392174617735601, 0.7001686266758796, 0.6761856794353899, 0.7935337130942207, 0.7461238039220504, 0.707828428633576, 0.6658974816262592, 0.7230469654910733, 0.6827783828717117, 0.6833371639497573, 0.6699082455766245, 1.531828804000543, 1.506129534025836, 1.4861906669857785, 1.4667056464850943, 2.381604727246217, 2.088076542348822, 1.5733202560839, 1.0158075253484642, 1.0726144808856672, 0.7654051526338639, 0.5062034224208162, 0.454484533913066 ], "peak_layer": 25, "peak_normalized": 2.381604727246217, "population_cv": 0.4360240290661341, "rest_mean": 1.70806682001933e-05, "spike_contrast": 1.913499224059756, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.2683845347492024e-05, "top_five_layers": [ 25, 26, 27, 21, 22 ], "top_five_spike_overlap": 3 }, "values": [ 1.701228757156059e-05, 1.9700953998835757e-05, 1.7643385945120826e-05, 1.6407509974669665e-05, 2.1202131392783485e-05, 1.9839759261230938e-05, 1.8739307051873766e-05, 1.6662383131915703e-05, 1.6038638932514004e-05, 1.4428537724597845e-05, 1.3666356608155183e-05, 1.3198241504142061e-05, 1.5488718418055214e-05, 1.4563340300810523e-05, 1.381586571369553e-05, 1.2997429621464107e-05, 1.4112911230768077e-05, 1.3326922271517105e-05, 1.3337828931980766e-05, 1.3075714377919212e-05, 2.9899252695031464e-05, 2.9397637263173237e-05, 2.9008457204326987e-05, 2.8628135623876005e-05, 4.648574395105243e-05, 4.075646575074643e-05, 3.0709110433235765e-05, 1.9827206415357068e-05, 2.0936002329108305e-05, 1.4939686479920056e-05, 9.880414836516138e-06, 8.870931196724996e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 7.340951196965761e-05, "first_to_last_ratio": 0.7646713125823572, "imbalance_abs_log_ratio": 0.26830919423960087, "last_quartile_mean": 9.600139401300112e-05, "mean": 7.792252529270627e-05, "normalized": [ 0.8806546012531954, 1.0193425393061946, 0.9114850055808843, 0.8419610576811557, 1.084269672716137, 1.0041793075648708, 0.9480869658790826, 0.8466874756345175, 0.8186780942474222, 0.7396812349765112, 0.7009835829373264, 0.6745560080882004, 0.8007781565162315, 0.7473372234399238, 0.7080296040910797, 0.664711327127518, 0.7283691776252745, 0.68335158262597, 0.683640529252796, 0.6665259156789554, 1.5327574162095816, 1.508583475650893, 1.4855945477407901, 1.4636689658272277, 2.3772369642506503, 2.0829939630291734, 1.568099944579317, 1.0123948412411625, 1.0730153673405642, 0.7688590083951055, 0.51224130260083, 0.4612451409114577 ], "peak_layer": 25, "peak_normalized": 2.3772369642506503, "population_cv": 0.43430966190612463, "rest_mean": 6.820286957749718e-05, "spike_contrast": 1.9120700780288336, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013040866615483538, "top_five_layers": [ 25, 26, 27, 21, 22 ], "top_five_spike_overlap": 3 }, "values": [ 6.862283044029027e-05, 7.942974480101839e-05, 7.102521340129897e-05, 6.560773181263357e-05, 8.448903099633753e-05, 7.824818749213591e-05, 7.387733057839796e-05, 6.597602623514831e-05, 6.379346450557932e-05, 5.7637829740997404e-05, 5.462241097120568e-05, 5.256310760159977e-05, 6.239865615498275e-05, 5.823440369567834e-05, 5.517145473277196e-05, 5.1795985200442374e-05, 5.675636566593312e-05, 5.3248480980983004e-05, 5.3270996431820095e-05, 5.193738252273761e-05, 0.00011943632853217423, 0.00011755263403756544, 0.00011576127872103825, 0.00011405278200982139, 0.00018524030747357756, 0.00016231214976869524, 0.0001221903075929731, 7.888836262281984e-05, 8.361206710105762e-05, 5.991143552819267e-05, 3.991513585788198e-05, 3.594138615881093e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.06510028941556811, "first_to_last_ratio": 0.7646713274805447, "imbalance_abs_log_ratio": 0.26830917475647514, "last_quartile_mean": 0.08513499470427632, "mean": 0.069102474139072, "normalized": [ 0.8806546433621915, 1.0193425588953635, 0.9114850878668271, 0.8419610449094133, 1.0842696180502707, 1.00417928339932, 0.9480869379318455, 0.8466875196948171, 0.8186780594337499, 0.7396812281537771, 0.700983532567485, 0.674556051029616, 0.8007781682987317, 0.7473372612208656, 0.7080296318375293, 0.6647113400213748, 0.7283691503414217, 0.6833515724004101, 0.6836405281324375, 0.6665258849811339, 1.5327574862719957, 1.5085834281667279, 1.485594627659287, 1.4636689254793798, 2.3772370822800757, 2.0829938916338477, 1.5680998990587263, 1.012394790829651, 1.073015331384321, 0.7688589644724497, 0.5122413142635286, 0.461245155971429 ], "peak_layer": 25, "peak_normalized": 2.3772370822800757, "population_cv": 0.4343096670813171, "rest_mean": 0.06048298585746023, "spike_contrast": 1.9120701337781434, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.11564771085977554, "top_five_layers": [ 25, 26, 27, 21, 22 ], "top_five_spike_overlap": 3 }, "values": [ 0.06085541471838951, 0.07043909281492233, 0.0629858747124672, 0.05818159133195877, 0.07492571324110031, 0.06939127296209335, 0.06551515311002731, 0.05850820243358612, 0.05657267943024635, 0.0511138029396534, 0.04843969643115997, 0.04661349207162857, 0.05533575266599655, 0.051642853766679764, 0.04892659932374954, 0.04593319818377495, 0.05033211037516594, 0.04722128435969353, 0.04724125191569328, 0.04605858772993088, 0.10591733455657959, 0.1042468473315239, 0.10265826433897018, 0.10114314407110214, 0.1642729640007019, 0.1439400315284729, 0.10835958272218704, 0.06995898485183716, 0.0741480141878128, 0.05313005670905113, 0.03539714217185974, 0.0318731814622879 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 2.8821277453516814e-06, "first_to_last_ratio": 0.4435988240760428, "imbalance_abs_log_ratio": 0.8128346743749869, "last_quartile_mean": 6.49714920086808e-06, "mean": 4.318623197718807e-06, "normalized": [ 0.7679289036988634, 0.6750658467666831, 0.6349650874581338, 0.6255999390265273, 0.6826544932425603, 0.6560057003168231, 0.6435433879931018, 0.6532117979760391, 0.6098414474176307, 0.5791837579115159, 0.5807235473785677, 0.5897432619493616, 0.6324444890605717, 0.6315023775676009, 0.6456861730412978, 0.6356409500509173, 0.6272694043965179, 0.6476401040643857, 0.6709176920607379, 0.7154473336085129, 1.6686289788439563, 1.74262953583039, 1.807292269514582, 1.840839946835644, 2.878409962831671, 2.5771886878268533, 1.9799066385490296, 1.2779514429489651, 1.2610217615467783, 0.9100450587895189, 0.613699555615541, 0.537370465880721 ], "peak_layer": 25, "peak_normalized": 2.878409962831671, "population_cv": 0.6226047324814435, "rest_mean": 3.5288268783780815e-06, "spike_contrast": 2.4324013667975355, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 8.583523322158725e-06, "top_five_layers": [ 25, 26, 27, 24, 23 ], "top_five_spike_overlap": 3 }, "values": [ 3.316395577712683e-06, 2.9153550258342875e-06, 2.742174956438248e-06, 2.7017304091714323e-06, 2.9481275305442978e-06, 2.833041435224004e-06, 2.7792214041255647e-06, 2.8209756237629335e-06, 2.633675421748194e-06, 2.5012764126586262e-06, 2.507926183170639e-06, 2.5468789317528717e-06, 2.7312894417264033e-06, 2.727220817178022e-06, 2.788475285342429e-06, 2.745093752309913e-06, 2.7089402010460617e-06, 2.796913577185478e-06, 2.897440708693466e-06, 3.0897474516677903e-06, 7.206179816421354e-06, 7.52576033846708e-06, 7.805014320183545e-06, 7.949894097691868e-06, 1.2430768038029782e-05, 1.1129906852147542e-05, 8.550470738555305e-06, 5.518990747077623e-06, 5.44587783224415e-06, 3.930141701857792e-06, 2.6503371373109985e-06, 2.3207005597214447e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 5.474985300679691e-05, "first_to_last_ratio": 0.9216218228319103, "imbalance_abs_log_ratio": 0.08162031003276306, "last_quartile_mean": 5.940598589404544e-05, "mean": 5.375807774044006e-05, "normalized": [ 0.961613559831088, 1.087275911775373, 0.9836256751130925, 0.917419537317466, 1.174351503605808, 1.0928129971118736, 1.0214427884182744, 0.9090489092459143, 0.8874531868957425, 0.8033400551634563, 0.7565003505971768, 0.7243654138759926, 0.8691310858172232, 0.804093934126664, 0.7545891253118167, 0.7041680219582052, 0.7861698865545709, 0.728808088925869, 0.7274781083815314, 0.7109016363744611, 1.5004530152121167, 1.4576742414199995, 1.4190259664069076, 1.3777644247831131, 2.258682820996343, 1.8741850598913514, 1.3996259672919458, 0.9390660822081557, 0.8796217108618816, 0.6253470664091118, 0.4515612501200868, 0.4124026179973873 ], "peak_layer": 25, "peak_normalized": 2.258682820996343, "population_cv": 0.38818123881913735, "rest_mean": 4.7757878915204116e-05, "spike_contrast": 1.8040824541159568, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 8.615915139671415e-05, "top_five_layers": [ 25, 26, 21, 22, 23 ], "top_five_spike_overlap": 4 }, "values": [ 5.1694496505660936e-05, 5.8449862990528345e-05, 5.2877825510222465e-05, 4.9318710807710886e-05, 6.313087942544371e-05, 5.8747526054503396e-05, 5.4910800827201456e-05, 4.88687219331041e-05, 4.7707777412142605e-05, 4.318601713748649e-05, 4.066800465807319e-05, 3.894049223163165e-05, 4.6722816477995366e-05, 4.322654422139749e-05, 4.05652608606033e-05, 3.7854719266761094e-05, 4.2262981878593564e-05, 3.9179321902338415e-05, 3.910782470484264e-05, 3.8216705434024334e-05, 8.066146983765066e-05, 7.836176519049332e-05, 7.628410821780562e-05, 7.406596705550328e-05, 0.00012142244668211788, 0.00010075258614961058, 7.524120155721903e-05, 5.0482387450756505e-05, 4.7286772314691916e-05, 3.3617456210777164e-05, 2.4275064788525924e-05, 2.216997199866455e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 3.9575083064846694e-05, "first_to_last_ratio": 1.1388638879593274, "imbalance_abs_log_ratio": 0.13003117597153577, "last_quartile_mean": 3.474961624760908e-05, "mean": 3.601960375476665e-05, "normalized": [ 1.0399564669309012, 1.1565841593540054, 1.059515615629035, 0.99505935539654, 1.278309320134729, 1.1938294123537594, 1.0939197395441878, 0.9725024266642768, 0.9661677087052931, 0.8822734997356141, 0.8263699672474701, 0.7897782850371386, 0.9587263341028692, 0.8854722697049854, 0.8269491515608597, 0.7684820883986734, 0.8593331751268791, 0.7911868305833119, 0.7901467329184265, 0.7817725731165415, 1.4617135450427476, 1.3702019185273646, 1.3035433540044417, 1.230271917612155, 2.0683207635967453, 1.6051472348290852, 1.2012992689470066, 0.8836170013027441, 0.7054407367103014, 0.4991106301888945, 0.3891269681251149, 0.36587154886790196 ], "peak_layer": 25, "peak_normalized": 2.0683207635967453, "population_cv": 0.33851512792424693, "rest_mean": 3.277243447680828e-05, "spike_contrast": 1.634126933525187, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.355431785574183e-05, "top_five_layers": [ 25, 26, 21, 22, 23 ], "top_five_spike_overlap": 4 }, "values": [ 3.7458819861058146e-05, 4.165970312897116e-05, 3.816333264694549e-05, 3.5841643693856895e-05, 4.604419518727809e-05, 4.3001262383768335e-05, 3.940255555789918e-05, 3.502915205899626e-05, 3.480097802821547e-05, 3.1779141863808036e-05, 2.976551877509337e-05, 2.8447500881156884e-05, 3.453294266364537e-05, 3.189436029060744e-05, 2.9786380764562637e-05, 2.768042031675577e-05, 3.095284046139568e-05, 2.8498236133600585e-05, 2.8460772227845155e-05, 2.8159138310002163e-05, 5.265034269541502e-05, 4.935413016937673e-05, 4.69531150883995e-05, 4.4313906983006746e-05, 7.450009434251115e-05, 5.781676736660302e-05, 4.327032365836203e-05, 3.182753425789997e-05, 2.5409695808775723e-05, 1.7977767129195854e-05, 1.4016199202160351e-05, 1.3178548215364572e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00015282425192708615, "first_to_last_ratio": 0.7534774818728942, "imbalance_abs_log_ratio": 0.2830561460696497, "last_quartile_mean": 0.00020282524110371014, "mean": 0.0001641354297134967, "normalized": [ 0.864522467755904, 0.9937702837504386, 0.893329661082966, 0.8209116086420106, 1.0708679702158068, 0.9974062360641608, 0.9576709717770711, 0.8502112970338779, 0.8207361545046119, 0.7364314167223136, 0.7023469902737517, 0.6647632777837263, 0.7917029719929976, 0.7419283443119661, 0.7044513316978172, 0.6445304634310127, 0.7188988583891809, 0.6668605708985835, 0.6651407834517624, 0.6530332067595712, 1.5682248611891052, 1.5500579550272269, 1.5260265009721472, 1.510425179362084, 2.4468289641633842, 2.0992714289663126, 1.6054021655104416, 1.045397065238643, 1.0538077815865954, 0.7204410639071442, 0.47870724212413573, 0.43589492541324876 ], "peak_layer": 25, "peak_normalized": 2.4468289641633842, "population_cv": 0.45565453072761564, "rest_mean": 0.00014224120133332217, "spike_contrast": 1.9851088174147118, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0002823642629664391, "top_five_layers": [ 25, 26, 27, 21, 22 ], "top_five_spike_overlap": 3 }, "values": [ 0.0001418987667420879, 0.0001631129125598818, 0.000146627047797665, 0.00013474067964125425, 0.00017576737445779145, 0.00016370970115531236, 0.00015718773647677153, 0.00013954979658592492, 0.0001347118814010173, 0.00012087448703823611, 0.00011528002505656332, 0.0001091112062567845, 0.0001299465075135231, 0.0001217767276102677, 0.00011562542204046622, 0.00010579028457868844, 0.00011799677304225042, 0.00010945544636342674, 0.00010917316831182688, 0.00010718588600866497, 0.0002574012614786625, 0.00025441942852921784, 0.0002504750154912472, 0.000247914285864681, 0.0004016113234683871, 0.000344564818078652, 0.0002635033742990345, 0.00017158669652417302, 0.0001729671930661425, 0.00011824990360764787, 7.857281889300793e-05, 7.154580089263618e-05 ] } } }, "mlp_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "c0b8018c349b1bd09e5d17858eb312a2323da0e92de9651e2bb08ae4adf115c7", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.13583384454250336, 0.0415157787501812, 0.018140379339456558, 0.039518751204013824, 0.022877270355820656, 0.015532293356955051, 0.02314678393304348, 0.05432315170764923, 0.03743875026702881, 0.04006272554397583, 0.045583780854940414, 0.05995568633079529, 0.038651175796985626, 0.04559725895524025, 0.05632574483752251, 0.07952121645212173, 0.042337141931056976, 0.05044656619429588, 0.04418695345520973, 0.06310147047042847, 0.029010163620114326, 0.04586305841803551, 0.06033914536237717, 0.07656749337911606, 0.045884523540735245, 0.051604680716991425, 0.06502768397331238, 0.06792232394218445, 0.060306549072265625, 0.08025261014699936, 0.09663844108581543, 0.12230555713176727 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.824558944463206e-05, "first_to_last_ratio": 0.9736472022607605, "imbalance_abs_log_ratio": 0.026706256293450907, "last_quartile_mean": 1.8739425740932347e-05, "mean": 1.796578632706769e-05, "normalized": [ 0.9452308711635502, 1.0952675913066698, 0.9826810745807577, 0.911503028916842, 1.175467085287599, 1.092237863694375, 1.0153068387276187, 0.9068994102561203, 0.8281892657330464, 0.7740730798865778, 0.7390295781204883, 0.7178707417012664, 0.838630050921232, 0.7854383552107032, 0.733427001850934, 0.6980610776954129, 0.779912828223095, 0.7273551438181467, 0.7256441637713558, 0.7473180457848747, 1.6522197939418646, 1.6170628444485469, 1.5986637594125668, 1.5680159647428409, 2.291381049228247, 1.7215960406287585, 1.1109239851486434, 0.8792448941297004, 0.8314955506562951, 0.5532980315979289, 0.4956673961453189, 0.4608875932686233 ], "peak_layer": 25, "peak_normalized": 2.291381049228247, "population_cv": 0.40574960171080005, "rest_mean": 1.5485613908418196e-05, "spike_contrast": 2.0250225514616456, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.135871738777496e-05, "top_five_layers": [ 25, 26, 21, 22, 23 ], "top_five_spike_overlap": 4 }, "values": [ 1.698181586107239e-05, 1.9677343516377732e-05, 1.765463821357116e-05, 1.6375868653994985e-05, 2.1118190488778055e-05, 1.9622912077466026e-05, 1.824078572099097e-05, 1.629316102480516e-05, 1.4879071386530995e-05, 1.3906831554777455e-05, 1.3277247489895672e-05, 1.2897112355858553e-05, 1.506664830230875e-05, 1.4111017662798986e-05, 1.317659280175576e-05, 1.2541216165118385e-05, 1.4011747225595172e-05, 1.3067507097730413e-05, 1.3036767995799892e-05, 1.3426156328932848e-05, 2.9683427783311345e-05, 2.905180554080289e-05, 2.872125151043292e-05, 2.8170639780000784e-05, 4.116646232432686e-05, 3.092982660746202e-05, 1.9958622942795046e-05, 1.579632589709945e-05, 1.4938471394998487e-05, 9.940434210875537e-06, 8.905054528440814e-06, 8.280208021460567e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 7.279734472831478e-05, "first_to_last_ratio": 0.9722740582023317, "imbalance_abs_log_ratio": 0.02811756137988487, "last_quartile_mean": 7.487327684430056e-05, "mean": 7.174056236181059e-05, "normalized": [ 0.9542356794294645, 1.105910924414439, 0.9908911472750251, 0.9126901248494818, 1.1731232477048164, 1.0786406989413262, 1.003603892046462, 0.8987491788511939, 0.8268959587862931, 0.7758336702656863, 0.7398166946433825, 0.7156966379734578, 0.847971628145978, 0.7871144601963379, 0.734707336971516, 0.6972341670292183, 0.7853129800731387, 0.7272537985352546, 0.7250684928511126, 0.740895554309, 1.6513842983385465, 1.6175579529721775, 1.595325178096626, 1.5647481399892045, 2.286540297040746, 1.7154977208455686, 1.1037042190227109, 0.876913972002831, 0.8353572611771873, 0.5596556498001242, 0.5030539810476304, 0.4686150563740623 ], "peak_layer": 25, "peak_normalized": 2.286540297040746, "population_cv": 0.4035876765363203, "rest_mean": 6.186811542310925e-05, "spike_contrast": 2.0212636990084873, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012505177583079785, "top_five_layers": [ 25, 26, 21, 22, 23 ], "top_five_spike_overlap": 4 }, "values": [ 6.84574042679742e-05, 7.933867163956165e-05, 7.108708814484999e-05, 6.547690281877294e-05, 8.416052151005715e-05, 7.738229032838717e-05, 7.199910760391504e-05, 6.447677151300013e-05, 5.932198109803721e-05, 5.565874380408786e-05, 5.3074865718372166e-05, 5.134447928867303e-05, 6.08339614700526e-05, 5.6468034017598256e-05, 5.2708317525684834e-05, 5.001997124054469e-05, 5.633879482047632e-05, 5.217359648668207e-05, 5.201682142796926e-05, 5.315226371749304e-05, 0.00011847123823827133, 0.00011604451719904318, 0.00011444952542660758, 0.00011225591151742265, 0.00016403768677264452, 0.00012307077122386545, 7.918036135379225e-05, 6.291030149441212e-05, 5.992899968987331e-05, 4.015001104562543e-05, 3.608937549870461e-05, 3.36187076754868e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.06455741357058287, "first_to_last_ratio": 0.972274060552552, "imbalance_abs_log_ratio": 0.02811755896264425, "last_quartile_mean": 0.06639837077818811, "mean": 0.06362024816917256, "normalized": [ 0.9542357116401934, 1.1059109410287413, 0.9908911486851065, 0.9126901621287556, 1.1731231838442338, 1.0786407747350526, 1.0036038132661569, 0.898749187114476, 0.8268959838033506, 0.7758337073304297, 0.7398166965930676, 0.7156966778269982, 0.8479716055610556, 0.7871144653788876, 0.7347072952590957, 0.6972341361757572, 0.7853129590248459, 0.7272538159700761, 0.7250684809041203, 0.740895515843762, 1.6513842988078489, 1.6175579516509786, 1.5953251639123671, 1.5647481566306345, 2.2865403037563077, 1.71549768836752, 1.1037042333495364, 0.8769139885374045, 0.8353572550009516, 0.5596556383802366, 0.5030539904388535, 0.46861506905319794 ], "peak_layer": 25, "peak_normalized": 2.2865403037563077, "population_cv": 0.4035876751437008, "rest_mean": 0.054865263471448864, "spike_contrast": 2.021263701660521, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.11089716553688049, "top_five_layers": [ 25, 26, 21, 22, 23 ], "top_five_spike_overlap": 4 }, "values": [ 0.06070871278643608, 0.07035832852125168, 0.06304074078798294, 0.05806557461619377, 0.07463438808917999, 0.06862339377403259, 0.06384952366352081, 0.057178646326065063, 0.052607327699661255, 0.049358732998371124, 0.047067321836948395, 0.04553280025720596, 0.053948163986206055, 0.05007641762495041, 0.046742260456085205, 0.044358208775520325, 0.04996180534362793, 0.04626806825399399, 0.04612903669476509, 0.04713595658540726, 0.10506147891283035, 0.10290943831205368, 0.10149498283863068, 0.09954966604709625, 0.1454702615737915, 0.10914038866758347, 0.07021793723106384, 0.055789485573768616, 0.0531456358730793, 0.035605430603027344, 0.03200441971421242, 0.029813406988978386 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 2.8555604387747735e-06, "first_to_last_ratio": 0.5552670584770549, "imbalance_abs_log_ratio": 0.5883060945236314, "last_quartile_mean": 5.142679356140434e-06, "mean": 3.931940092627428e-06, "normalized": [ 0.8353389686459259, 0.7403666831894464, 0.7000151632346908, 0.6882346333445044, 0.7527807695249172, 0.7132879857158784, 0.6931716433405022, 0.6867816633514217, 0.6326028144574801, 0.6197141390615448, 0.6233711984953005, 0.6330342064875842, 0.6783960615179854, 0.669708485263476, 0.6701667091839245, 0.6654832142898003, 0.6838231004768655, 0.6928556747007584, 0.7095730411048928, 0.7769119491477562, 1.8260867426548284, 1.8949709131411827, 1.968680313356217, 1.981250822673113, 2.85659660244116, 2.204164847537207, 1.484073936726756, 1.111747455718133, 0.9904477657177253, 0.6698328140657178, 0.598627280273134, 0.5479024011601699 ], "peak_layer": 25, "peak_normalized": 2.85659660244116, "population_cv": 0.5846469936655838, "rest_mean": 3.12697214347416e-06, "spike_contrast": 2.6475346239756132, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 8.278767018055077e-06, "top_five_layers": [ 25, 26, 24, 23, 22 ], "top_five_spike_overlap": 4 }, "values": [ 3.2845027817529626e-06, 2.911077444878174e-06, 2.7524176857696148e-06, 2.706097347981995e-06, 2.95988888865395e-06, 2.8046056286257226e-06, 2.725509375522961e-06, 2.7003843570128083e-06, 2.487356368874316e-06, 2.4366788693441777e-06, 2.451058207952883e-06, 2.4890525764931226e-06, 2.66741267296311e-06, 2.6332536435802467e-06, 2.635055352584459e-06, 2.616640131236636e-06, 2.6887514650297817e-06, 2.724267005760339e-06, 2.7899986889678985e-06, 3.0547712412953842e-06, 7.1800636760599446e-06, 7.450912107742624e-06, 7.740733053651638e-06, 7.790159543219488e-06, 1.1231966709601693e-05, 8.666644134791568e-06, 5.835289812239353e-06, 4.371324394014664e-06, 3.8943812796787824e-06, 2.6337424969824497e-06, 2.353766603846452e-06, 2.1543194179685088e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 5.436818855741876e-05, "first_to_last_ratio": 1.1493256256670847, "imbalance_abs_log_ratio": 0.1391753579047175, "last_quartile_mean": 4.730442560685333e-05, "mean": 5.014859789298498e-05, "normalized": [ 1.0287494960573844, 1.1641554435011037, 1.0552498839455033, 0.9819809734736323, 1.2546264899615047, 1.1599491985400199, 1.0705047944632766, 0.9579177005430451, 0.8909240618429074, 0.8333807272212836, 0.788792368823235, 0.7601401814450041, 0.9100229326931617, 0.8376466033326904, 0.7746386033550339, 0.7304968982325775, 0.8374163487449479, 0.7683188621244832, 0.7642162825086684, 0.7851208455276361, 1.5974436517594142, 1.5439201240741427, 1.5032302065364247, 1.4548764529346048, 2.1049362250922856, 1.5460978943085557, 1.023594520945546, 0.8307274312078374, 0.6761892426837561, 0.49154947436310764, 0.4466772151052196, 0.4265088646520061 ], "peak_layer": 25, "peak_normalized": 2.1049362250922856, "population_cv": 0.36527191905507306, "rest_mean": 4.419687563010181e-05, "spike_contrast": 1.8618487605605563, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 8.22878981125541e-05, "top_five_layers": [ 25, 21, 26, 22, 23 ], "top_five_spike_overlap": 4 }, "values": [ 5.159034481039271e-05, 5.8380763221066445e-05, 5.291930210660212e-05, 4.924496897729114e-05, 6.291775935096666e-05, 5.8169825933873653e-05, 5.36843144800514e-05, 4.803822957910597e-05, 4.467859253054485e-05, 4.179287498118356e-05, 3.9556831325171515e-05, 3.811996430158615e-05, 4.5636374125024304e-05, 4.200680268695578e-05, 3.884703983203508e-05, 3.66333952115383e-05, 4.199525574222207e-05, 3.853011367027648e-05, 3.832437505479902e-05, 3.93727095797658e-05, 8.010955934878439e-05, 7.742542948108166e-05, 7.538488716818392e-05, 7.296001422218978e-05, 0.00010555960034253076, 7.753464160487056e-05, 5.133183003636077e-05, 4.165981590631418e-05, 3.390994243090972e-05, 2.4650516934343614e-05, 2.2400236048270017e-05, 2.138882155122701e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 3.942232751796837e-05, "first_to_last_ratio": 1.388212756932184, "imbalance_abs_log_ratio": 0.3280171334303115, "last_quartile_mean": 2.8397900337040483e-05, "mean": 3.41086660853307e-05, "normalized": [ 1.0952348595734618, 1.221109155982299, 1.118831983813601, 1.0494967127932664, 1.346091574101062, 1.251617537031842, 1.136826444253296, 1.0270817912519055, 0.9694994325180898, 0.9054586889217315, 0.8523390280514652, 0.8208390430240674, 0.9914834537614479, 0.9103137833685365, 0.8354024065116694, 0.7829445615694002, 0.9030531133471876, 0.8259388120460255, 0.8216805785070482, 0.8549182969763304, 1.534338262011861, 1.4355767669775124, 1.3599460852667513, 1.2894063199233112, 1.8559893356975876, 1.3243363353179058, 0.957425478177861, 0.8171160217859265, 0.5287226117070379, 0.4209442079615098, 0.3873691553542305, 0.3686681624147717 ], "peak_layer": 25, "peak_normalized": 1.8559893356975876, "population_cv": 0.3263791641479537, "rest_mean": 3.098171398758293e-05, "spike_contrast": 1.6459453287060375, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.0994207413168624e-05, "top_five_layers": [ 25, 21, 22, 23, 5 ], "top_five_spike_overlap": 4 }, "values": [ 3.735700011020526e-05, 4.165040445514023e-05, 3.8161866541486233e-05, 3.579693293431774e-05, 4.59133880212903e-05, 4.269100463716313e-05, 3.8775633584009483e-05, 3.503238986013457e-05, 3.3068332413677126e-05, 3.088398807449266e-05, 2.9072147299302742e-05, 2.799772482831031e-05, 3.381817805347964e-05, 3.104958886979148e-05, 2.8494461730588228e-05, 2.670519461389631e-05, 3.080193710047752e-05, 2.8171671146992594e-05, 2.8026428481098264e-05, 2.9160122721805237e-05, 5.233423144090921e-05, 4.8965608584694564e-05, 4.638594691641629e-05, 4.3979929614579305e-05, 6.330532050924376e-05, 4.5171345846029e-05, 3.265650593675673e-05, 2.7870737540069968e-05, 1.8034023014479317e-05, 1.435784542991314e-05, 1.3212645171734039e-05, 1.2574779248097911e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00015136150796024594, "first_to_last_ratio": 0.9620596966080336, "imbalance_abs_log_ratio": 0.03867877555563735, "last_quartile_mean": 0.00015733068175904918, "mean": 0.00015084571123225032, "normalized": [ 0.9362483215057325, 1.077264296126485, 0.9735787834592439, 0.8900224650975405, 1.1680277072581131, 1.0755535171951405, 1.0199979634818723, 0.8866618755603617, 0.8277663766479834, 0.7762227702786177, 0.7354871501702283, 0.7076604153623335, 0.8366976575221102, 0.7859695989753221, 0.7280859620349116, 0.6789371953503681, 0.7802934671640332, 0.7130594403460433, 0.7120984657027258, 0.7353596665592114, 1.6935389796801803, 1.6586054797266194, 1.6413257828877592, 1.617610647765939, 2.2864964621986568, 1.758047905638686, 1.1417270134965103, 0.9208024014068031, 0.7843452561510307, 0.5258666155027939, 0.4744277260912481, 0.4522126336553954 ], "peak_layer": 25, "peak_normalized": 2.2864964621986568, "population_cv": 0.42017041582571274, "rest_mean": 0.0001290704213143154, "spike_contrast": 2.0797350318970933, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.000268432276789099, "top_five_layers": [ 25, 26, 21, 22, 23 ], "top_five_spike_overlap": 4 }, "values": [ 0.00014122904394753277, 0.00016250069893430918, 0.00014686018403153867, 0.00013425607176031917, 0.00017619197024032474, 0.00016224263526964933, 0.0001538623182568699, 0.00013374914124142379, 0.00012486500781960785, 0.00011708987585734576, 0.00011094508226960897, 0.00010674753866624087, 0.00012621225323528051, 0.00011856014316435903, 0.00010982864478137344, 0.00010241476411465555, 0.00011770392302423716, 0.00010756195842986926, 0.00010741699952632189, 0.00011092585191363469, 0.0002554630918893963, 0.00025019352324306965, 0.0002475869550835341, 0.00024400962865911424, 0.00034490818507038057, 0.0002651939867064357, 0.00017222462338395417, 0.00013889909314457327, 0.00011831511801574379, 7.932472362881526e-05, 7.156538777053356e-05, 6.821433635195717e-05 ] } } }, "post_attention_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "90a633f12d984935028fe5d7212be48e7da7b4a0806751939a551938d16dc540", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.0933690071105957, 0.35791492462158203, 0.3940892517566681, 0.4057151973247528, 0.09380833059549332, 0.11286329478025436, 0.13231655955314636, 0.1621972620487213, 0.05156813934445381, 0.08533531427383423, 0.12003090232610703, 0.16063299775123596, 0.046871740370988846, 0.079384446144104, 0.11459959298372269, 0.15625107288360596, 0.08397205173969269, 0.11654024571180344, 0.15294702351093292, 0.21572710573673248, 0.03708760067820549, 0.06553087383508682, 0.09390194714069366, 0.12852425873279572, 0.025487789884209633, 0.06059965491294861, 0.08860157430171967, 0.12100546807050705, 0.024166326969861984, 0.06877554953098297, 0.12682415544986725, 0.2025238573551178 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.8401236047793645e-05, "first_to_last_ratio": 0.7650956611689681, "imbalance_abs_log_ratio": 0.2677544056936637, "last_quartile_mean": 2.4050895831351227e-05, "mean": 1.951877015926584e-05, "normalized": [ 0.8716100826030018, 1.0093579132463284, 0.9039242474578353, 0.8406066604718192, 1.0862553167234708, 1.016461831275563, 0.9600785532354511, 0.8536707535194554, 0.821709226451063, 0.7392171928068182, 0.700157991560061, 0.6761821167747123, 0.7935250427753293, 0.7461295084869718, 0.7078306591692773, 0.6658958384256345, 0.7230375113669247, 0.6827738060531082, 0.6833471686273892, 0.6699096560210135, 1.5318345940068188, 1.5060888670752761, 1.4861573767970278, 1.4666924978815465, 2.381588682386594, 2.0881248957489498, 1.5733110042664489, 1.0158202057048318, 1.072592466334956, 0.765419699365803, 0.5062130801010698, 0.4544755532794493 ], "peak_layer": 25, "peak_normalized": 2.381588682386594, "population_cv": 0.4360227747025414, "rest_mean": 1.708083093237494e-05, "spike_contrast": 1.9134690878842584, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.2683641984476707e-05, "top_five_layers": [ 25, 26, 27, 21, 22 ], "top_five_spike_overlap": 3 }, "values": [ 1.7012756870826706e-05, 1.9701425117091276e-05, 1.764348962751683e-05, 1.6407608200097457e-05, 2.1202367861405946e-05, 1.9840084860334173e-05, 1.8739552615443245e-05, 1.6662603229633532e-05, 1.6038753528846428e-05, 1.4428610484173987e-05, 1.3666222912434023e-05, 1.3198243323131464e-05, 1.5488632925553247e-05, 1.4563530385203194e-05, 1.381598394800676e-05, 1.2997467820241582e-05, 1.4112803000898566e-05, 1.3326904991117772e-05, 1.3338096323423088e-05, 1.3075812603347003e-05, 2.98995273624314e-05, 2.93970024358714e-05, 2.9007964258198626e-05, 2.8628033760469407e-05, 4.648568210541271e-05, 4.07576299039647e-05, 3.0709095881320536e-05, 1.982756111829076e-05, 2.0935685824952088e-05, 1.4940051187295467e-05, 9.88065676210681e-06, 8.870803867466748e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 7.340988304349594e-05, "first_to_last_ratio": 0.7646728959142083, "imbalance_abs_log_ratio": 0.26830712363725734, "last_quartile_mean": 9.600168050383218e-05, "mean": 7.79226685381218e-05, "normalized": [ 0.8806689493118375, 1.019347295006326, 0.9114681100206553, 0.8419614707579783, 1.0842873814320044, 1.0041852116285845, 0.9480891447201258, 0.8466833046927548, 0.8186771495141438, 0.7396826297542543, 0.7009762716893748, 0.674553134003956, 0.8007809796517145, 0.7473409851827282, 0.708035912506332, 0.6647082843926093, 0.7283601353008534, 0.6833492059302504, 0.683647162624784, 0.6665341678721638, 1.532767204039224, 1.5085646420764787, 1.4855738889453405, 1.4636595522274218, 2.3772184013102216, 2.083022441288971, 1.568089965507483, 1.0124065193953047, 1.0729985483350373, 0.7688685664544674, 0.5122492781712971, 0.4612441062553222 ], "peak_layer": 25, "peak_normalized": 2.3772184013102216, "population_cv": 0.43430786523087517, "rest_mean": 6.820316142424147e-05, "spike_contrast": 1.9120522308621983, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013040800695307553, "top_five_layers": [ 25, 26, 27, 21, 22 ], "top_five_spike_overlap": 3 }, "values": [ 6.86240746290423e-05, 7.9430261394009e-05, 7.102402742020786e-05, 6.560788460774347e-05, 8.449056622339413e-05, 7.824879139661789e-05, 7.387763616861776e-05, 6.597582250833511e-05, 6.379350816132501e-05, 5.763804438174702e-05, 5.462194167193957e-05, 5.256298027234152e-05, 6.2398990849033e-05, 5.823480387334712e-05, 5.517204772331752e-05, 5.17958433192689e-05, 5.675576539942995e-05, 5.3248393669491634e-05, 5.327161125023849e-05, 5.193812103243545e-05, 0.00011943731078645214, 0.0001175513825728558, 0.00011575988173717633, 0.00011405225814087316, 0.0001852392015280202, 0.00016231466725002974, 0.00012218975462019444, 7.88894176366739e-05, 8.361091022379696e-05, 5.9912290453212336e-05, 3.991583071183413e-05, 3.59413716068957e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.06510061863809824, "first_to_last_ratio": 0.7646728941608533, "imbalance_abs_log_ratio": 0.2683071259302051, "last_quartile_mean": 0.08513525081798434, "mean": 0.06910260207951069, "normalized": [ 0.8806689161864696, 1.0193472485926036, 0.9114681977985657, 0.8419614267993061, 1.0842874492876218, 1.004185187180837, 0.9480891718972634, 0.8466832566124041, 0.8186771366922776, 0.7396826619630942, 0.7009763046765162, 0.6745531309225539, 0.8007809445476378, 0.7473409989660956, 0.7080359221984898, 0.6647082764127086, 0.7283601466415279, 0.6833492290138943, 0.6836471331965127, 0.6665341929283229, 1.5327671554546884, 1.508564677863339, 1.4855738713556907, 1.463659530776114, 2.377218448808177, 2.0830223807929755, 1.5680899875487804, 1.0124065016320982, 1.0729985735263077, 0.768868592421875, 0.5122492609472172, 0.4612440863580343 ], "peak_layer": 25, "peak_normalized": 2.377218448808177, "population_cv": 0.43430786478426203, "rest_mean": 0.06048324593791255, "spike_contrast": 1.9120522295191478, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.11564712524414063, "top_five_layers": [ 25, 26, 27, 21, 22 ], "top_five_spike_overlap": 3 }, "values": [ 0.06085651367902756, 0.07043954730033875, 0.06298482418060303, 0.058181725442409515, 0.07492708414793015, 0.06939180940389633, 0.0655154287815094, 0.0585080161690712, 0.056572720408439636, 0.05111399665474892, 0.04843928664922714, 0.04661337658762932, 0.05533604696393013, 0.05164320766925812, 0.048927124589681625, 0.0459330715239048, 0.05033158138394356, 0.04722120985388756, 0.04724179580807686, 0.046059247106313705, 0.10591819882392883, 0.10424574464559555, 0.1026570200920105, 0.10114268213510513, 0.1642719805240631, 0.14394226670265198, 0.10835909843444824, 0.06995992362499237, 0.07414699345827103, 0.05313082039356232, 0.03539775684475899, 0.03187316656112671 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 2.882070589294017e-06, "first_to_last_ratio": 0.4435911485782981, "imbalance_abs_log_ratio": 0.8128519773156906, "last_quartile_mean": 6.497132773120029e-06, "mean": 4.318578383788463e-06, "normalized": [ 0.7679633028655034, 0.6750218339641407, 0.6349780998118787, 0.6256088001387001, 0.6826621563062814, 0.6560003455164206, 0.6435386409728591, 0.6531515001068428, 0.6098858417859744, 0.5791486483623859, 0.5807670604388429, 0.5897333234340136, 0.6324532105982226, 0.6315514719719342, 0.6456979277447012, 0.6356286973599049, 0.6272635407988052, 0.6476330829504716, 0.6709368690083494, 0.7154723429530178, 1.6685893267921041, 1.74257885805589, 1.8072000373486798, 1.8408470450222292, 2.8784771084059826, 2.577195845494106, 1.9798532632894768, 1.277992503527661, 1.2609768267484005, 0.910061030951073, 0.6137257730775662, 0.5374056841975803 ], "peak_layer": 25, "peak_normalized": 2.8784771084059826, "population_cv": 0.622599618306022, "rest_mean": 3.5288075853006576e-06, "spike_contrast": 2.4323629124400963, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 8.58334069562261e-06, "top_five_layers": [ 25, 26, 27, 24, 23 ], "top_five_spike_overlap": 3 }, "values": [ 3.3165097192977555e-06, 2.915134700742783e-06, 2.742202696026652e-06, 2.701740640986827e-06, 2.9481300316547276e-06, 2.8329889119049767e-06, 2.7791720640379936e-06, 2.820685949700419e-06, 2.6338398129155394e-06, 2.501098833818105e-06, 2.5080880732275546e-06, 2.5468095827818615e-06, 2.7312987640470965e-06, 2.7274045351077802e-06, 2.7884971132152714e-06, 2.745012352534104e-06, 2.7088867682323325e-06, 2.7968542326561874e-06, 2.897493459386169e-06, 3.0898233944753883e-06, 7.205933798104525e-06, 7.525463388446951e-06, 7.804535016475711e-06, 7.949842256493866e-06, 1.2430929018591996e-05, 1.1129822269140277e-05, 8.550151505914982e-06, 5.519110800378257e-06, 5.445627266453812e-06, 3.930169896193547e-06, 2.6504228571866406e-06, 2.3208285711007193e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 5.474991985465749e-05, "first_to_last_ratio": 0.9216217946223237, "imbalance_abs_log_ratio": 0.08162034064139916, "last_quartile_mean": 5.940606024523731e-05, "mean": 5.375812509100797e-05, "normalized": [ 0.9616198185096786, 1.0872764429025619, 0.983623049227553, 0.9174198796899647, 1.1743591313844182, 1.0928146061315054, 1.0214368132413851, 0.9090439128166732, 0.8874567362983851, 0.8033412424221015, 0.7564960299186483, 0.7243671444077843, 0.8691312677032689, 0.804094443990318, 0.7545922503573663, 0.7041633413358109, 0.786164592319532, 0.7288067025820621, 0.7274850470000972, 0.7109023636687979, 1.5004669877215138, 1.457662265140068, 1.4190167310925474, 1.3777633465838597, 2.2586783952997447, 1.8741935600593416, 1.3996171551038343, 0.939069653822862, 0.8796215451422201, 0.6253485457915371, 0.45156423603632634, 0.4124027622982324 ], "peak_layer": 25, "peak_normalized": 2.2586783952997447, "population_cv": 0.38818080155368334, "rest_mean": 4.7757946352552004e-05, "spike_contrast": 1.8040786268873175, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 8.615909027867019e-05, "top_five_layers": [ 25, 26, 21, 22, 23 ], "top_five_spike_overlap": 4 }, "values": [ 5.169487849343568e-05, 5.84499430260621e-05, 5.287773092277348e-05, 4.9318772653350607e-05, 6.313134508673102e-05, 5.8747664297698066e-05, 5.4910527978790924e-05, 4.886849637841806e-05, 4.770801024278626e-05, 4.3186119000893086e-05, 4.0667808207217604e-05, 3.89406195608899e-05, 4.6722867409698665e-05, 4.322660970501602e-05, 4.05654645874165e-05, 3.785450098803267e-05, 4.226273449603468e-05, 3.917928188457154e-05, 3.9108232158469036e-05, 3.8216778193600476e-05, 8.066229202086106e-05, 7.83611903898418e-05, 7.628367893630639e-05, 7.406597433146089e-05, 0.00012142231571488082, 0.00010075313184643164, 7.524079410359263e-05, 5.0482623919378966e-05, 4.728680505650118e-05, 3.3617565350141376e-05, 2.427524668746628e-05, 2.2169999283505604e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 3.9575365917698946e-05, "first_to_last_ratio": 1.1388142153243417, "imbalance_abs_log_ratio": 0.12998755906630063, "last_quartile_mean": 3.475138032627001e-05, "mean": 3.602119954848604e-05, "normalized": [ 1.0399658418093327, 1.1564747475193156, 1.0593656620790157, 0.9948984209739714, 1.2782189566002862, 1.1937174415384906, 1.0942463745032434, 0.9724624741935212, 0.9664902067100636, 0.8821534152973619, 0.8261901966944667, 0.7897832403799221, 0.9589113029238555, 0.8856963371920366, 0.8269145363703108, 0.7684706159704388, 0.8594587180638286, 0.7911755642465123, 0.7903281110646931, 0.7817129935202365, 1.4615536511634988, 1.3700974854577694, 1.3036480058114828, 1.2300816767594935, 2.0682673101493507, 1.605017950970488, 1.2015181314273413, 0.8836221927521581, 0.7054901800156923, 0.4990788232643951, 0.38918547583057606, 0.3658039587468507 ], "peak_layer": 25, "peak_normalized": 2.0682673101493507, "population_cv": 0.3384893534695339, "rest_mean": 3.277442455227073e-05, "spike_contrast": 1.6340114360402496, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.3553784528048706e-05, "top_five_layers": [ 25, 26, 21, 22, 23 ], "top_five_spike_overlap": 4 }, "values": [ 3.746081711142324e-05, 4.1657607653178275e-05, 3.815962190856226e-05, 3.583743455237709e-05, 4.604298010235652e-05, 4.2999134166166186e-05, 3.9416067011188716e-05, 3.502926483633928e-05, 3.481413659756072e-05, 3.177622420480475e-05, 2.9760361940134317e-05, 2.844893970177509e-05, 3.454113539191894e-05, 3.1903844501357526e-05, 2.978645352413878e-05, 2.7681233405019157e-05, 3.095873398706317e-05, 2.8499092877609655e-05, 2.846856659743935e-05, 2.815823972923681e-05, 5.264691571937874e-05, 4.935255492455326e-05, 4.69589649583213e-05, 4.4309017539490014e-05, 7.450146949850023e-05, 5.781467189081013e-05, 4.3280124373268336e-05, 3.182913133059628e-05, 2.541260255384259e-05, 1.7977417883230373e-05, 1.401892768626567e-05, 1.3176697393646464e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00015282826643669978, "first_to_last_ratio": 0.753562132698821, "imbalance_abs_log_ratio": 0.28294380552438825, "last_quartile_mean": 0.0002028077842624043, "mean": 0.00016413739194831578, "normalized": [ 0.8644588497282479, 0.9939011410317065, 0.8935569366443981, 0.8207805121190377, 1.0711454309180837, 0.9974876679751449, 0.9573625223083662, 0.8501040535765721, 0.8209700606088056, 0.7363641435718767, 0.7023838531904392, 0.6646453960731447, 0.7917805684106393, 0.7419904888748008, 0.7043701671001704, 0.6445838427943074, 0.7187864024957232, 0.6669537562359544, 0.6650746728680259, 0.6529782343776496, 1.5682167521556694, 1.5503366023336795, 1.526390900839116, 1.510595429776903, 2.446241528783042, 2.0991349794422063, 1.6052079644432122, 1.0453846563500786, 1.0537012958167107, 0.7204317862289332, 0.4787441189087413, 0.4359352840186137 ], "peak_layer": 25, "peak_normalized": 2.446241528783042, "population_cv": 0.4556092057916065, "rest_mean": 0.00014224157806623866, "spike_contrast": 1.9851775461886172, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0002823747869115323, "top_five_layers": [ 25, 26, 27, 21, 22 ], "top_five_spike_overlap": 3 }, "values": [ 0.00014189002104103565, 0.00016313634114339948, 0.00014666610513813794, 0.00013472077262122184, 0.00017581501742824912, 0.00016372502432204783, 0.0001571389875607565, 0.00013953386223874986, 0.0001347518846159801, 0.000120864890050143, 0.0001152874538092874, 0.00010909316188190132, 0.00012996079749427736, 0.00012178838369436562, 0.00011561348219402134, 0.00010580031084828079, 0.00011797972547356039, 0.00010947205009870231, 0.00010916362225543708, 0.00010717814438976347, 0.0002574030077084899, 0.0002544682065490633, 0.0002505378215573728, 0.00024794519413262606, 0.00040151970461010933, 0.0003445465408731252, 0.0002634746488183737, 0.00017158671107608825, 0.00017295178258791566, 0.00011824979446828365, 7.857981108827516e-05, 7.155328057706356e-05 ] } } }, "post_mlp_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "89f2226456bfe96e245d2aad0f699d32c676f9f53099789cd870253d002402f5", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.16915453970432281, 0.374446302652359, 0.39966297149658203, 0.4194087088108063, 0.09892481565475464, 0.11664740741252899, 0.13951337337493896, 0.18593810498714447, 0.06607726216316223, 0.10425819456577301, 0.14551182091236115, 0.19865906238555908, 0.06484340131282806, 0.10430748760700226, 0.14720691740512848, 0.20439592003822327, 0.10052042454481125, 0.14085541665554047, 0.1745898574590683, 0.2420409768819809, 0.049456026405096054, 0.08509506285190582, 0.11783231794834137, 0.15649692714214325, 0.05533855780959129, 0.08507220447063446, 0.11942370235919952, 0.1559232622385025, 0.06628894805908203, 0.12384890764951706, 0.19928614795207977, 0.30097496509552 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.8245657201987342e-05, "first_to_last_ratio": 0.9736554372168914, "imbalance_abs_log_ratio": 0.02669779848523559, "last_quartile_mean": 1.873933683782525e-05, "mean": 1.796578885659983e-05, "normalized": [ 0.9452202083481644, 1.095268753312293, 0.9826724314404816, 0.9115131265671594, 1.175469855959754, 1.092244898475752, 1.015325325296821, 0.9069081923384195, 0.8281806949684788, 0.7740691741217974, 0.7390252723001353, 0.7178607690056398, 0.8386305909525956, 0.7854417376584957, 0.7334304422454775, 0.6980529808658235, 0.7799159583303671, 0.7273429929680517, 0.725662842995341, 0.7473178393172129, 1.652217637613409, 1.6170854986829928, 1.5986788226825814, 1.5680101753636602, 2.2913701968787326, 1.7215741312894015, 1.1109184626215167, 0.8792407204385277, 0.8314893081164029, 0.5533110652333831, 0.4956623652341742, 0.46088752837695673 ], "peak_layer": 25, "peak_normalized": 2.2913701968787326, "population_cv": 0.40574893029850145, "rest_mean": 1.548560349975661e-05, "spike_contrast": 2.0250285876198557, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.135878978355322e-05, "top_five_layers": [ 25, 26, 21, 22, 23 ], "top_five_spike_overlap": 4 }, "values": [ 1.6981626686174423e-05, 1.9677367163239978e-05, 1.7654485418461263e-05, 1.6376052371924743e-05, 2.1118243239470758e-05, 1.9623041225713678e-05, 1.8241120415041223e-05, 1.629332109587267e-05, 1.48789195009158e-05, 1.3906763342674822e-05, 1.3277172001835424e-05, 1.2896935004391707e-05, 1.5066660125739872e-05, 1.4111080417933408e-05, 1.3176656466384884e-05, 1.2541072464955505e-05, 1.4011805433256086e-05, 1.3067290637991391e-05, 1.303710541833425e-05, 1.3426154509943444e-05, 2.9683393222512677e-05, 2.905221663240809e-05, 2.8721526177832857e-05, 2.817053973558359e-05, 4.116627314942889e-05, 3.092943734372966e-05, 1.9958526536356658e-05, 1.579625313752331e-05, 1.4938361346139573e-05, 9.940669770003296e-06, 8.90496539796004e-06, 8.280208021460567e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 7.279763212864054e-05, "first_to_last_ratio": 0.9722801406690796, "imbalance_abs_log_ratio": 0.02811130548148771, "last_quartile_mean": 7.487310404030723e-05, "mean": 7.17406516059782e-05, "normalized": [ 0.9542276972169219, 1.1059234432593774, 0.9908809896376276, 0.9126887866376932, 1.1731221940403236, 1.0786585255649925, 1.0036098444205637, 0.8987553630830893, 0.8268904676488731, 0.7758318430691542, 0.7398151658020864, 0.7156921472385842, 0.847975137196494, 0.7871158644159381, 0.7347128631962562, 0.6972308655946716, 0.7853130173598494, 0.7272519810611239, 0.7250822968199613, 0.7408976245458778, 1.6513767673485684, 1.6175712552208372, 1.595334958292372, 1.5647324003094618, 2.2865295418404212, 1.7154842277228097, 1.1036914869608094, 0.8769176478901343, 0.8353554106444507, 0.5596671747426969, 0.5030485377931891, 0.4686144734247902 ], "peak_layer": 25, "peak_normalized": 2.2865295418404212, "population_cv": 0.40358563480445514, "rest_mean": 6.186822146345448e-05, "spike_contrast": 2.021260211100045, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012505177437560633, "top_five_layers": [ 25, 26, 21, 22, 23 ], "top_five_spike_overlap": 4 }, "values": [ 6.845691677881405e-05, 7.93396684457548e-05, 7.108644786057994e-05, 6.547688826685771e-05, 8.416055061388761e-05, 7.738366548437625e-05, 7.199962419690564e-05, 6.447729538194835e-05, 5.932166095590219e-05, 5.565868195844814e-05, 5.307482206262648e-05, 5.1344220992177725e-05, 6.083428888814524e-05, 5.646820500260219e-05, 5.2708779548993334e-05, 5.001979661756195e-05, 5.6338867580052465e-05, 5.2173531003063545e-05, 5.201787644182332e-05, 5.315247835824266e-05, 0.00011847084533656016, 0.00011604561586864293, 0.00011445036943769082, 0.00011225492198718712, 0.0001640371192479506, 0.00012306995631661266, 7.917954644653946e-05, 6.291064346441999e-05, 5.9928941482212394e-05, 4.015088779851794e-05, 3.608902989071794e-05, 3.36187076754868e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.06455766828730702, "first_to_last_ratio": 0.9722801401073566, "imbalance_abs_log_ratio": 0.028111306059225523, "last_quartile_mean": 0.06639821757562459, "mean": 0.06362032698234543, "normalized": [ 0.9542277371445754, 1.1059235071185238, 0.9908810207993924, 0.9126887387085484, 1.173122199013742, 1.078658527450089, 1.0036098308208337, 0.8987553345635776, 0.8268904507030695, 0.7758318678983468, 0.7398151945541953, 0.7156921608064463, 0.8479751223821903, 0.7871158324989204, 0.7347128847086777, 0.6972308716823696, 0.7853129816108516, 0.7272519781639685, 0.7250823385538606, 0.740897642880443, 1.651376748891854, 1.6175711721206463, 1.5953348986221858, 1.564732399225337, 2.2865297419147153, 1.7154842035243185, 1.103691389289001, 0.8769176451696703, 0.8353554003858383, 0.5596671830778299, 0.5030485071854275, 0.46861448853055365 ], "peak_layer": 25, "peak_normalized": 2.2865297419147153, "population_cv": 0.4035856447520544, "rest_mean": 0.05486535715560118, "spike_contrast": 2.021260223135956, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.11089716404676438, "top_five_layers": [ 25, 26, 21, 22, 23 ], "top_five_spike_overlap": 4 }, "values": [ 0.06070828065276146, 0.07035921514034271, 0.06304017454385757, 0.05806555598974228, 0.07463441789150238, 0.06862460821866989, 0.06384998559951782, 0.05717910826206207, 0.05260704085230827, 0.04935867711901665, 0.04706728458404541, 0.04553256928920746, 0.053948454558849335, 0.05007656663656235, 0.046742673963308334, 0.04435805603861809, 0.049961868673563004, 0.04626800864934921, 0.0461299754679203, 0.04713615030050278, 0.1050611287355423, 0.10291040688753128, 0.10149572789669037, 0.09954878687858582, 0.1454697698354721, 0.10913966596126556, 0.07021720707416534, 0.05578978732228279, 0.05314558371901512, 0.03560620918869972, 0.032004110515117645, 0.029813406988978386 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 2.8555820961173595e-06, "first_to_last_ratio": 0.5552711869155001, "imbalance_abs_log_ratio": 0.5882986595010874, "last_quartile_mean": 5.1426801235265884e-06, "mean": 3.931962893943819e-06, "normalized": [ 0.8352956117525471, 0.7404263465068224, 0.6999415957995699, 0.6882394898289709, 0.7528127195404866, 0.7132878394476022, 0.6932198414563248, 0.686764438342056, 0.6326045239280614, 0.6197193350669655, 0.6234650799338666, 0.6330303042361277, 0.6783843208758006, 0.6697013055073787, 0.6701920833797892, 0.6654571496021477, 0.6837868675363192, 0.6928592900206609, 0.7096184840669101, 0.7769191249192576, 1.8260372934873976, 1.8949396848155198, 1.968673060592436, 1.9812902212411339, 2.8565328502927154, 2.2041201451497097, 1.4841192254110436, 1.1117850729182932, 0.9904218983444383, 0.6698565132144071, 0.5985990588920833, 0.5478992238931567 ], "peak_layer": 25, "peak_normalized": 2.8565328502927154, "population_cv": 0.5846353651548142, "rest_mean": 3.1270066284816024e-06, "spike_contrast": 2.64749254192011, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 8.27872672743979e-06, "top_five_layers": [ 25, 26, 24, 23, 22 ], "top_five_spike_overlap": 4 }, "values": [ 3.2843513508851174e-06, 2.911328920163214e-06, 2.752144382611732e-06, 2.7061321361543378e-06, 2.960031679322128e-06, 2.804621317409328e-06, 2.725714693951886e-06, 2.700332288441132e-06, 2.4873775146261323e-06, 2.436713430142845e-06, 2.4514415599696804e-06, 2.489051666998421e-06, 2.667381977516925e-06, 2.6332406832807465e-06, 2.635170403664233e-06, 2.616552819745266e-06, 2.6886245905188844e-06, 2.7242970190854976e-06, 2.7901935482077533e-06, 3.0548171707778238e-06, 7.179910880950047e-06, 7.450832526956219e-06, 7.74074942455627e-06, 7.790359632053878e-06, 1.1231781172682531e-05, 8.666518624522723e-06, 5.835501724504866e-06, 4.371497652755352e-06, 3.894302153639728e-06, 2.633850954225636e-06, 2.3536692879133625e-06, 2.1543194179685088e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 5.436835226646508e-05, "first_to_last_ratio": 1.1493259099181605, "imbalance_abs_log_ratio": 0.1391756052245669, "last_quartile_mean": 4.730455634671671e-05, "mean": 5.014869481101414e-05, "normalized": [ 1.0287440257782987, 1.1641650908284165, 1.0552455956966695, 0.981975230860819, 1.2546234849030928, 1.15995914417173, 1.070506715502695, 0.9579240467123875, 0.8909239359957548, 0.833379044074858, 0.7887919325502717, 0.7601350851961468, 0.9100237130051773, 0.8376478136955079, 0.7746381944358244, 0.7304974451482692, 0.8374180673596266, 0.768315273490004, 0.7642219148703521, 0.7851226652092942, 1.5974365020637946, 1.5439281669371643, 1.5032248348833948, 1.4548649359605836, 2.1049421681144533, 1.5460956317377519, 1.0235881901029735, 0.8307325723100063, 0.6761931590278627, 0.4915528407464875, 0.44667453825498377, 0.4265080403753478 ], "peak_layer": 25, "peak_normalized": 2.1049421681144533, "population_cv": 0.36527170977389195, "rest_mean": 4.419697971671768e-05, "spike_contrast": 1.8618456927971772, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 8.228795632021501e-05, "top_five_layers": [ 25, 21, 26, 22, 23 ], "top_five_spike_overlap": 4 }, "values": [ 5.159017018740997e-05, 5.838135984959081e-05, 5.29191893292591e-05, 4.924477616441436e-05, 6.29177302471362e-05, 5.8170437114313245e-05, 5.368451456888579e-05, 4.8038640670711175e-05, 4.46786725660786e-05, 4.179287134320475e-05, 3.955688589485362e-05, 3.8119782402645797e-05, 4.563650145428255e-05, 4.2006944568129256e-05, 3.8847094401717186e-05, 3.663349343696609e-05, 4.19954230892472e-05, 3.853000816889107e-05, 3.8324731576722115e-05, 3.937287692679092e-05, 8.010935562197119e-05, 7.742598245386034e-05, 7.538476347690448e-05, 7.295957766473293e-05, 0.00010556010238360614, 7.753467798465863e-05, 5.1331611757632345e-05, 4.166015423834324e-05, 3.3910204365383834e-05, 2.4650733394082636e-05, 2.240014509879984e-05, 2.138882155122701e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 3.942640523746377e-05, "first_to_last_ratio": 1.3883433156419591, "imbalance_abs_log_ratio": 0.32811117706311327, "last_quartile_mean": 2.839816693267494e-05, "mean": 3.4110308263279876e-05, "normalized": [ 1.0953632289052508, 1.2210249844332346, 1.1189651896931818, 1.0494176036084724, 1.3461649917498881, 1.2517558687579076, 1.1366892707953324, 1.0274201358890724, 0.9695338143036344, 0.905510658687136, 0.8522391744341097, 0.8207868334913778, 0.9915387477578653, 0.9103504812713468, 0.8353363774900436, 0.782885324168264, 0.9030301149628676, 0.8258958491429226, 0.8217061854091349, 0.854833517287873, 1.5345522515273924, 1.435534850384316, 1.3596606939504845, 1.2894906788875813, 1.8560004499705036, 1.324350007920428, 0.9572487343325576, 0.8170843622819884, 0.528722594156077, 0.42098996080470885, 0.38726664997974947, 0.36865041356529754 ], "peak_layer": 25, "peak_normalized": 1.8560004499705036, "population_cv": 0.32639873885862847, "rest_mean": 3.0983228161521426e-05, "spike_contrast": 1.6459402018059863, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.0996540812775494e-05, "top_five_layers": [ 25, 21, 22, 23, 5 ], "top_five_spike_overlap": 4 }, "values": [ 3.7363177398219705e-05, 4.1649538616184145e-05, 3.816824755631387e-05, 3.579595795599744e-05, 4.591810284182429e-05, 4.269777855370194e-05, 3.87728214263916e-05, 3.504561755107716e-05, 3.3071097277570516e-05, 3.088724770350382e-05, 2.907014095399063e-05, 2.7997291908832267e-05, 3.382169234100729e-05, 3.105233554379083e-05, 2.849358133971691e-05, 2.6704459742177278e-05, 3.080263559240848e-05, 2.817156200762838e-05, 2.8028651286149397e-05, 2.9158634788473137e-05, 5.234405034570955e-05, 4.896653626929037e-05, 4.6378445404116064e-05, 4.398492455948144e-05, 6.330874748528004e-05, 4.517398701864295e-05, 3.265204941271804e-05, 2.7870999474544078e-05, 1.8034890672424808e-05, 1.436009733879473e-05, 1.3209784810896963e-05, 1.2574779248097911e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00015137388254515827, "first_to_last_ratio": 0.9621529335160675, "imbalance_abs_log_ratio": 0.03858186640261841, "last_quartile_mean": 0.00015732829706394114, "mean": 0.0001508458642547339, "normalized": [ 0.9363782798709707, 1.077509681029368, 0.9735028395967293, 0.8903139590793369, 1.1680272941266088, 1.0756747485244147, 1.0201698008046631, 0.8864264605206594, 0.8280285108088443, 0.7761649215769858, 0.7354287639793783, 0.7078267814032271, 0.8362840188776303, 0.786065752780109, 0.7280961244142795, 0.6789818951735448, 0.7801085649607451, 0.7129761879634445, 0.7121832146607415, 0.7356361718383577, 1.6932887581487153, 1.6586624502015577, 1.6408080099504283, 1.617665730451809, 2.2864181253173266, 1.7579720342065541, 1.1418340932567865, 0.9208478687968995, 0.784271481862393, 0.5258472224033733, 0.4743880784965249, 0.45221217491759275 ], "peak_layer": 25, "peak_normalized": 2.2864181253173266, "population_cv": 0.4201270657436487, "rest_mean": 0.00012907465457424728, "spike_contrast": 2.0794973065371605, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0002684103965293616, "top_five_layers": [ 25, 26, 21, 22, 23 ], "top_five_spike_overlap": 4 }, "values": [ 0.00014124879089649767, 0.00016253787907771766, 0.00014684887719340622, 0.00013430017861537635, 0.00017619208665564656, 0.0001622610870981589, 0.00015388839528895915, 0.00013371376553550363, 0.00012490467634052038, 0.00011708126839948818, 0.00011093638750026003, 0.0001067727425834164, 0.00012614998559001833, 0.00011857476783916354, 0.00010983028914779425, 0.0001024216107907705, 0.00011767615069402382, 0.0001075495092663914, 0.00010742989252321422, 0.00011096767411800101, 0.00025542560615576804, 0.0002502023708075285, 0.00024750910233706236, 0.00024401818518526852, 0.0003448967181611806, 0.0002651828108355403, 0.0001722409506328404, 0.00013890609261579812, 0.00011830410949187353, 7.932187872938812e-05, 7.155947969295084e-05, 6.821433635195717e-05 ] } } }, "pre_attention_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "ee695e78937de01b7128d3866b8fc5dd328ebdf2c159be37918d4f02952d5248", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.0626930519938469, 0.09030644595623016, 0.20229491591453552, 0.20300470292568207, 0.19049881398677826, 0.12695962190628052, 0.12132706493139267, 0.11585728824138641, 0.13427142798900604, 0.10485932976007462, 0.10419609397649765, 0.10852954536676407, 0.1286265105009079, 0.09096142649650574, 0.10080574452877045, 0.10566592961549759, 0.1287592649459839, 0.09650235623121262, 0.1065765842795372, 0.11446042358875275, 0.12395420670509338, 0.07221997529268265, 0.07677769660949707, 0.08416415005922318, 0.09094838052988052, 0.059439923614263535, 0.062182098627090454, 0.06896103918552399, 0.0776243805885315, 0.06285123527050018, 0.07231374084949493, 0.08016443997621536 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.0946848533421871e-05, "first_to_last_ratio": 1.269689556482922, "imbalance_abs_log_ratio": 0.23877242687503095, "last_quartile_mean": 8.621673288189413e-06, "mean": 1.1776095099946815e-05, "normalized": [ 0.544517981490592, 1.145130850906001, 0.8274821968170263, 0.743675670893471, 0.47361194984984256, 1.005104706921629, 1.3113477747850195, 1.3857865654851524, 0.9632489804296396, 0.7976974126630908, 0.6135617497231088, 0.5817761860625408, 0.5554752343171182, 0.9752238458084466, 0.8452544262383666, 0.7644801941845483, 0.8754285383276427, 1.2322730391043752, 1.2985316953868253, 1.5832466507057747, 1.2439039118975828, 2.396192264570633, 1.9485403631719076, 2.0314401048683584, 1.361415455395021, 1.3812289340389468, 0.8821955536751157, 0.6400118370057548, 0.5243003076969728, 0.3203633108894651, 0.35806828490675446, 0.38948402178327596 ], "peak_layer": 22, "peak_normalized": 2.396192264570633, "population_cv": 0.49543729127313324, "rest_mean": 1.0039560670015311e-05, "spike_contrast": 2.1070026584683883, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.1153381021576932e-05, "top_five_layers": [ 22, 24, 23, 20, 8 ], "top_five_spike_overlap": 3 }, "values": [ 6.412295533664292e-06, 1.3485169802152086e-05, 9.74450904323021e-06, 8.757595423958264e-06, 5.577299361902988e-06, 1.1836208614113275e-05, 1.5442556104972027e-05, 1.631915438338183e-05, 1.1343311598466244e-05, 9.393760592502076e-06, 7.225361514429096e-06, 6.851051693956833e-06, 6.541329184983624e-06, 1.1484328751976136e-05, 9.953796507033985e-06, 9.002591468743049e-06, 1.0309129720553756e-05, 1.4511364497593604e-05, 1.5291632735170424e-05, 1.864446312538348e-05, 1.46483307617018e-05, 2.8217787985340692e-05, 2.2946196622797288e-05, 2.3922431864775717e-05, 1.603215787326917e-05, 1.6265483282040805e-05, 1.0388818736828398e-05, 7.536840257671429e-06, 6.174210284370929e-06, 3.7726288155681686e-06, 4.216646175336791e-06, 4.586600880429614e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 4.5063141214995994e-05, "first_to_last_ratio": 1.3230598364138495, "imbalance_abs_log_ratio": 0.27994711193611027, "last_quartile_mean": 3.405979077797383e-05, "mean": 4.7540331649997825e-05, "normalized": [ 0.7108324538776775, 1.173897475148572, 0.8100371634394925, 0.7499283624272937, 0.4695974587003249, 1.0280808942305133, 1.2562445482666185, 1.3845245713523466, 0.9366002163724741, 0.780823341381619, 0.6497928908220746, 0.5954270668577573, 0.5541579517578961, 0.9897660666787615, 0.8458778153448299, 0.7799078841485974, 0.8785085128446201, 1.3196335527640195, 1.304334405161488, 1.5786079327732951, 1.2386102692199965, 2.31888691835071, 1.9285062016388992, 1.9858967918031367, 1.3235446970572349, 1.3901910243483164, 0.8653392591789515, 0.5659983270000892, 0.5131153478516941, 0.3256237300004076, 0.35643062977098244, 0.3912762394293097 ], "peak_layer": 22, "peak_normalized": 2.31888691835071, "population_cv": 0.48193637738177425, "rest_mean": 4.085749060322996e-05, "spike_contrast": 2.046813743767542, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 8.362767330254428e-05, "top_five_layers": [ 22, 24, 23, 20, 26 ], "top_five_spike_overlap": 3 }, "values": [ 3.379321060492657e-05, 5.580747529165819e-05, 3.850943539873697e-05, 3.5651843063533306e-05, 2.2324818928609602e-05, 4.8875306674744934e-05, 5.972228245809674e-05, 6.582075729966164e-05, 4.4526284909807146e-05, 3.7120600609341636e-05, 3.089136953349225e-05, 2.830680023180321e-05, 2.6344852813053876e-05, 4.7053807065822184e-05, 4.021331187686883e-05, 3.70770794688724e-05, 4.1764586057979614e-05, 6.273581675486639e-05, 6.200849020387977e-05, 7.504754466935992e-05, 5.888394298381172e-05, 0.00011024065315723419, 9.168182441499084e-05, 9.44101921049878e-05, 6.292175385169685e-05, 6.609014235436916e-05, 4.113851537113078e-05, 2.690774817892816e-05, 2.4393673811573535e-05, 1.5480260117328726e-05, 1.6944830349530093e-05, 1.860140218923334e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.03996244492009282, "first_to_last_ratio": 1.3230598687629405, "imbalance_abs_log_ratio": 0.2799471363863201, "last_quartile_mean": 0.030204562819562852, "mean": 0.0421592420025263, "normalized": [ 0.7108324530319086, 1.1738974552346684, 0.8100372328283234, 0.7499282927333294, 0.4695974550672632, 1.0280809097581924, 1.2562445515412406, 1.3845246138456817, 0.9366002376705983, 0.7808233981331483, 0.6497928805166009, 0.5954270964674194, 0.5541579402360114, 0.9897660166320406, 0.84587780303066, 0.7799078756771219, 0.8785085219891383, 1.3196335909596741, 1.3043343544881636, 1.5786079520846796, 1.2386103395943417, 2.3188868095544777, 1.9285063024024847, 1.985896774361232, 1.323544685926245, 1.3901909388157763, 0.8653392603079487, 0.5659983203211986, 0.5131153451065483, 0.32562372878787665, 0.3564306264575799, 0.3912762364384265 ], "peak_layer": 22, "peak_normalized": 2.3188868095544777, "population_cv": 0.4819363732459248, "rest_mean": 0.0362328316119534, "spike_contrast": 2.0468137546046377, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.07416185811161995, "top_five_layers": [ 22, 24, 23, 20, 26 ], "top_five_spike_overlap": 3 }, "values": [ 0.029968157410621643, 0.04949062690138817, 0.03415055572986603, 0.03161640837788582, 0.019797872751951218, 0.043343111872673035, 0.05296231806278229, 0.05837050825357437, 0.0394863560795784, 0.03291892260313034, 0.027394775301218033, 0.025102755054831505, 0.02336287871003151, 0.041727785021066666, 0.035661567002534866, 0.03288032487034798, 0.03703725337982178, 0.0556347519159317, 0.05498974770307541, 0.06655291467905045, 0.05221887305378914, 0.09776251018047333, 0.08130436390638351, 0.08372390270233154, 0.05579964071512222, 0.05860939621925354, 0.03648204728960991, 0.02386206015944481, 0.02163255400955677, 0.013728049583733082, 0.015026845037937164, 0.01649590954184532 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 3.4114774649651736e-06, "first_to_last_ratio": 0.5621001560042546, "imbalance_abs_log_ratio": 0.5760752314358888, "last_quartile_mean": 6.069162992616839e-06, "mean": 4.157112524438844e-06, "normalized": [ 2.9949099263655197, 1.0189598239989015, 0.2757579146470541, 0.3290816421082844, 0.24262255793963006, 0.5026335625419994, 0.5060193531714852, 0.6951056658479873, 0.4895657004225874, 0.449426171553459, 0.43887513169970604, 0.5308132425797975, 0.371340141102806, 0.6290280808229218, 0.6982372336636214, 0.7095097838979514, 0.4190991619821391, 0.8168725960025041, 0.6703984134743438, 1.0417832131243765, 0.8271840439810254, 1.7122830860048508, 1.9006358280317452, 2.0502835842105305, 1.8431020548195274, 2.5553359280556616, 2.06933782476713, 1.6029845596995627, 1.3301227504727833, 0.976654369263052, 0.6362087267171761, 0.6658279270298789 ], "peak_layer": 1, "peak_normalized": 2.9949099263655197, "population_cv": 0.7036274008106085, "rest_mean": 3.64386485048103e-06, "spike_contrast": 1.9014563514605605, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.92864996381104e-06, "top_five_layers": [ 1, 26, 27, 24, 23 ], "top_five_spike_overlap": 2 }, "values": [ 1.2450177564460319e-05, 4.2359306462458335e-06, 1.1463566806924064e-06, 1.3680294159712503e-06, 1.0086092743222252e-06, 2.089504278046661e-06, 2.1035793906776235e-06, 2.88963246930507e-06, 2.035179704762413e-06, 1.868315166575485e-06, 1.824453306653595e-06, 2.2066503788664704e-06, 1.5437027514053625e-06, 2.614940513012698e-06, 2.9026507490925724e-06, 2.9495120088540716e-06, 1.7422423752577743e-06, 3.3958312997128814e-06, 2.7869216410181252e-06, 4.330810043029487e-06, 3.4386971492494922e-06, 7.1181534622155596e-06, 7.901157005107962e-06, 8.52325956657296e-06, 7.661982635909226e-06, 1.0622818990668748e-05, 8.60247018863447e-06, 6.663787189609138e-06, 5.529469945031451e-06, 4.060062110511353e-06, 2.6447912659932626e-06, 2.7679216145770624e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 2.9569775961135747e-05, "first_to_last_ratio": 1.3352184817357151, "imbalance_abs_log_ratio": 0.28909493519278373, "last_quartile_mean": 2.2146020569380198e-05, "mean": 3.182846225513458e-05, "normalized": [ 0.6323471544336242, 1.1362781966813003, 0.8138038569540852, 0.7435815301644464, 0.5194394333394948, 1.0605253720262944, 1.2823471880537642, 1.2439624284924289, 0.9506784784925476, 0.8262581641587752, 0.6954707364277555, 0.6310108212836854, 0.6030680682580712, 1.0521861914005077, 0.876674665500747, 0.7550958378972896, 0.8696811905877263, 1.3399114115469932, 1.2913741090518884, 1.5105537767584425, 1.1934766616251726, 2.323606228492693, 2.0783056853847346, 2.004018981014916, 1.3061277802370188, 1.465265210278452, 0.8413126717507515, 0.5151579437704077, 0.5235336431013567, 0.3258519484812045, 0.27325072560410635, 0.31584390874931767 ], "peak_layer": 22, "peak_normalized": 2.323606228492693, "population_cv": 0.4940709285653522, "rest_mean": 2.7224492475431395e-05, "spike_contrast": 2.0823124293939106, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.668989906553179e-05, "top_five_layers": [ 22, 23, 24, 20, 26 ], "top_five_spike_overlap": 3 }, "values": [ 2.0126637537032366e-05, 3.616598769440316e-05, 2.5902125344146043e-05, 2.36670566664543e-05, 1.653295839787461e-05, 3.3754891774151474e-05, 4.0815139072947204e-05, 3.959341120207682e-05, 3.0258634069468826e-05, 2.629852679092437e-05, 2.2135764083941467e-05, 2.0084104107809253e-05, 1.9194729247828946e-05, 3.348946847836487e-05, 2.790320650092326e-05, 2.4033539375523105e-05, 2.768061494862195e-05, 4.2647319787647575e-05, 4.110245208721608e-05, 4.807860386790708e-05, 3.7986526876920834e-05, 7.39568131393753e-05, 6.614927406189963e-05, 6.378484249580652e-05, 4.157203875365667e-05, 4.663713843910955e-05, 2.6777688617585227e-05, 1.6396685168729164e-05, 1.6663270798744634e-05, 1.0371366442996077e-05, 8.697150406078435e-06, 1.0052825928141829e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 1.9050893797611934e-05, "first_to_last_ratio": 1.3291034177161873, "imbalance_abs_log_ratio": 0.28450459289334284, "last_quartile_mean": 1.433364292324768e-05, "mean": 2.0942702391835155e-05, "normalized": [ 0.5547365080336302, 1.1314106370578119, 0.7938696572726529, 0.7318934633907598, 0.5883748710255681, 1.1067566095342585, 1.3187732902707623, 1.0515242260072302, 0.9757273204786265, 0.9041035623422258, 0.737292750224254, 0.6641973989339395, 0.6651564142092118, 1.1323694351942726, 0.896642933342537, 0.7476212617919106, 0.8675425978017992, 1.3410201169290192, 1.3119630354395504, 1.426520174247898, 1.100116765242383, 2.305571563341637, 2.232325430860554, 1.9391152502832156, 1.2453437189216472, 1.556038437741431, 0.8305564754682776, 0.472863841417902, 0.5334737110417407, 0.33671005469484416, 0.23555131425854417, 0.2648371731999052 ], "peak_layer": 22, "peak_normalized": 2.305571563341637, "population_cv": 0.49991162729948563, "rest_mean": 1.7977779845279184e-05, "spike_contrast": 2.0554976454971454, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.69532841432374e-05, "top_five_layers": [ 22, 23, 24, 26, 20 ], "top_five_spike_overlap": 3 }, "values": [ 1.1617681593634188e-05, 2.3694796254858375e-05, 1.6625775970169343e-05, 1.532782698632218e-05, 1.2322159818722866e-05, 2.317847429367248e-05, 2.761867654044181e-05, 2.202175892307423e-05, 2.043436688836664e-05, 1.8934371837531216e-05, 1.5440902643604204e-05, 1.3910088455304503e-05, 1.3930172826803755e-05, 2.3714876078884117e-05, 1.8778126104734838e-05, 1.5657209587516263e-05, 1.8168686438002624e-05, 2.8084585210308433e-05, 2.7476051400299184e-05, 2.9875187465222552e-05, 2.303941801073961e-05, 4.828489909414202e-05, 4.6750927140237764e-05, 4.0610313590150326e-05, 2.6080862880917266e-05, 3.2587649911874905e-05, 1.7394097085343674e-05, 9.903046702675056e-06, 1.117238116421504e-05, 7.051618467812659e-06, 4.933081072522327e-06, 5.546406100620516e-06 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 8.919242691263207e-05, "first_to_last_ratio": 1.3537187982346888, "imbalance_abs_log_ratio": 0.30285547067201657, "last_quartile_mean": 6.588696783182968e-05, "mean": 9.402016343074138e-05, "normalized": [ 0.6926033304501219, 1.133450684620324, 0.8404837777172331, 0.7569225446590931, 0.4667337997038146, 1.0439601213408876, 1.2533791831722163, 1.4016834801583533, 0.8889619859402359, 0.7437027206766162, 0.670677832897842, 0.605490643803241, 0.5477590927642155, 1.0026259902881585, 0.8322763229699769, 0.7797835761265279, 0.8897945948734107, 1.3427568385202926, 1.2608298684467039, 1.535465526510342, 1.3195615717166387, 2.3156960137428357, 2.0124440378133786, 2.056757498852008, 1.3677156049400065, 1.4064404716187364, 0.8473728640486917, 0.5321624368764438, 0.5189389370017533, 0.29715709513094235, 0.2843955001330888, 0.35201605248586904 ], "peak_layer": 22, "peak_normalized": 2.3156960137428357, "population_cv": 0.4997264265703907, "rest_mean": 7.983992145454977e-05, "spike_contrast": 2.1366938618456595, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00017059347010217607, "top_five_layers": [ 22, 24, 23, 20, 26 ], "top_five_spike_overlap": 3 }, "values": [ 6.511867832159624e-05, 0.00010656721860868856, 7.902242214186117e-05, 7.116598135326058e-05, 4.388238812680356e-05, 9.815330122364685e-05, 0.0001178429156425409, 0.00013178650988265872, 8.35803512018174e-05, 6.992305134190246e-05, 6.305723945843056e-05, 5.6928329286165535e-05, 5.150039942236617e-05, 9.426705946680158e-05, 7.825075590517372e-05, 7.331537926802412e-05, 8.365863322978839e-05, 0.00012624621740542352, 0.00011854343028971925, 0.00014436471974477172, 0.00012406539462972432, 0.00021772211766801775, 0.00018921031733043492, 0.00019337667617946863, 0.00012859284470323473, 0.00013223376299720258, 7.96701351646334e-05, 5.003399928682484e-05, 4.879072366748005e-05, 2.7938758648815565e-05, 2.6738911401480436e-05, 3.309660678496584e-05 ] } } }, "pre_mlp_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "e53fa425d89dfeb0e9395eca9bcd73dc0a55314abf45c5b10b43e2cbfa724345", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.059514161199331284, 0.08157797157764435, 0.14292451739311218, 0.10997853428125381, 0.0991024449467659, 0.1065012738108635, 0.12155740708112717, 0.1095810979604721, 0.06964563578367233, 0.08311934769153595, 0.09363401681184769, 0.0941167026758194, 0.08564268052577972, 0.09493856877088547, 0.09063401818275452, 0.09473779052495956, 0.10505300015211105, 0.09355910867452621, 0.09618847072124481, 0.11416846513748169, 0.047205954790115356, 0.053352247923612595, 0.06512638926506042, 0.07829347252845764, 0.05368548259139061, 0.05310077592730522, 0.061235878616571426, 0.07320034503936768, 0.05304349213838577, 0.0639304369688034, 0.07404901832342148, 0.08157160878181458 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 5.563098426364377e-06, "first_to_last_ratio": 0.1761679651237226, "imbalance_abs_log_ratio": 1.7363173917499597, "last_quartile_mean": 3.15783770474809e-05, "mean": 1.60104112367776e-05, "normalized": [ 0.7354847016189815, 0.2664094346130556, 0.175290361589128, 0.31537381510860996, 0.25084006350579974, 0.20359404986381396, 0.27657992025895983, 0.5561680831367258, 0.5863803688803918, 0.4349692112232699, 0.3987287665155364, 0.40085418002076023, 0.4451827845651173, 0.46718295878785127, 0.5225477509455817, 0.5305774001337921, 0.387368015222834, 0.4995918081213406, 0.4586337577643135, 0.43229038923567564, 1.389404877672481, 1.9908715071655363, 2.177074885772482, 2.3196797647950045, 3.0198765807430648, 3.0538277537267904, 2.588067313776065, 1.711307515434275, 1.942251089753615, 1.5376061013931355, 1.0005151595424926, 0.9254696291135193 ], "peak_layer": 26, "peak_normalized": 3.0538277537267904, "population_cv": 0.8648292090528146, "rest_mean": 1.2513673608635952e-05, "spike_contrast": 2.7883733841884957, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.4892794428742493e-05, "top_five_layers": [ 26, 25, 27, 24, 23 ], "top_five_spike_overlap": 3 }, "values": [ 1.1775412531278562e-05, 4.265324605512433e-06, 2.8064707748853834e-06, 5.04926447320031e-06, 4.016052571387263e-06, 3.259624463680666e-06, 4.428158263181103e-06, 8.904479727789294e-06, 9.388190846948419e-06, 6.96403594702133e-06, 6.383811523846816e-06, 6.41784026811365e-06, 7.127559456421295e-06, 7.47979129300802e-06, 8.366204383492004e-06, 8.49476236908231e-06, 6.2019212236918975e-06, 7.99867029854795e-06, 7.3429150688753e-06, 6.921146905369824e-06, 2.2244943465921097e-05, 3.187467154930346e-05, 3.485586421447806e-05, 3.7139026971999556e-05, 4.8349465942010283e-05, 4.88930381834507e-05, 4.1436022002017125e-05, 2.739873707469087e-05, 3.1096238672034815e-05, 2.4617706003482454e-05, 1.6018659152905457e-05, 1.4817149349255487e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 2.2405845243156364e-05, "first_to_last_ratio": 0.1772762282429712, "imbalance_abs_log_ratio": 1.7300461513548109, "last_quartile_mean": 0.00012638945145226899, "mean": 6.403039063229699e-05, "normalized": [ 0.7495901933430097, 0.27428428709734054, 0.17646910508078795, 0.3206538877590326, 0.2484906747282376, 0.20195836598830444, 0.27405790199467733, 0.5538969335115237, 0.5856955037656872, 0.4345374122180075, 0.39776282979574407, 0.4022168409271472, 0.4430037150729445, 0.46766864046577544, 0.5260109137215828, 0.531083144529753, 0.3871673876919175, 0.4995389982910809, 0.46205202294379744, 0.4257246427158023, 1.391771180717727, 1.9827121486152186, 2.16925347320969, 2.3032168648126894, 3.0289843796600646, 3.0476081266877375, 2.5949245096149482, 1.717341309750644, 1.94653348541727, 1.5307649862295216, 1.0043995379262802, 0.920626595716055 ], "peak_layer": 26, "peak_normalized": 3.0476081266877375, "population_cv": 0.8640033430275056, "rest_mean": 5.009562735520174e-05, "spike_contrast": 2.780244897245492, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013927811232861131, "top_five_layers": [ 26, 25, 27, 24, 23 ], "top_five_spike_overlap": 3 }, "values": [ 4.799655289389193e-05, 1.756253004714381e-05, 1.1299385732854716e-05, 2.053159369097557e-05, 1.5910954971332103e-05, 1.2931473065691534e-05, 1.7548034520586953e-05, 3.5466237022774294e-05, 3.750231189769693e-05, 2.782360024866648e-05, 2.546890937082935e-05, 2.5754101443453692e-05, 2.8365700927679427e-05, 2.9945005735498853e-05, 3.368068428244442e-05, 3.400546120246872e-05, 2.479047907399945e-05, 3.1985677196644247e-05, 2.9585371521534398e-05, 2.7259315174887888e-05, 8.911565237212926e-05, 0.00012695383338723332, 0.00013889814727008343, 0.00014747587556485087, 0.0001939470530487597, 0.00019513953884597868, 0.00016615403001196682, 0.00010996203491231427, 0.00012463729945011437, 9.801548003451899e-05, 6.431209476431832e-05, 5.894808055018075e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.019869727548211813, "first_to_last_ratio": 0.1772762253474888, "imbalance_abs_log_ratio": 1.7300461676879788, "last_quartile_mean": 0.11208343086764216, "mean": 0.05678279104176909, "normalized": [ 0.7495901722956588, 0.27428429054236114, 0.17646911318627773, 0.3206538875236958, 0.24849067870632507, 0.2019583625086003, 0.2740578843269545, 0.553896934257654, 0.5856954936738917, 0.43453739478075487, 0.3977628190736026, 0.40221684143438596, 0.4430037144810719, 0.467668641144736, 0.5260109182978651, 0.5310831129462952, 0.38716738870722034, 0.49953900629924164, 0.46205201532315265, 0.4257246427360581, 1.391771162853645, 1.9827122477645847, 2.169253386858678, 2.303216848895015, 3.028984245609035, 3.0476082070129094, 2.5949246263321046, 1.7173412012172098, 1.9465335711169913, 1.530765028533135, 1.004399575553132, 0.9206265860077576 ], "peak_layer": 26, "peak_normalized": 3.0476082070129094, "population_cv": 0.8640033471803245, "rest_mean": 0.0444253038752962, "spike_contrast": 2.7802448372087616, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.12351322174072266, "top_five_layers": [ 26, 25, 27, 24, 23 ], "top_five_spike_overlap": 3 }, "values": [ 0.042563822120428085, 0.015574627555906773, 0.010020408779382706, 0.018207622691988945, 0.014109994284808636, 0.011467759497463703, 0.01556177157908678, 0.031451813876628876, 0.03325742483139038, 0.024674246087670326, 0.02258608303964138, 0.02283899486064911, 0.02515498735010624, 0.026555530726909637, 0.029868368059396744, 0.03015638142824173, 0.021984444931149483, 0.02836521901190281, 0.026236603036522865, 0.024173833429813385, 0.0790286511182785, 0.1125839352607727, 0.12317626178264618, 0.1307830810546875, 0.1719941794872284, 0.17305169999599457, 0.14734706282615662, 0.09751542657613754, 0.11052960902452469, 0.08692111074924469, 0.05703261122107506, 0.05227574706077576 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 1.2989937658858253e-06, "first_to_last_ratio": 0.1528938346293502, "imbalance_abs_log_ratio": 1.8780114897551383, "last_quartile_mean": 8.49605066832737e-06, "mean": 4.1625307147796775e-06, "normalized": [ 0.7210817360032282, 0.23234099760979796, 0.17836161535432038, 0.3436050818242345, 0.2642336551941004, 0.1840408625853127, 0.21945688483928208, 0.3534253398560146, 0.4892198173377355, 0.3912439262188246, 0.36613725077122894, 0.39629977815286876, 0.4019316937716108, 0.4630989398766952, 0.5434481890006447, 0.5158171244837687, 0.371207659975515, 0.4882814333021366, 0.45562409569487883, 0.3303795969697513, 1.3730255605488746, 2.042569971487586, 2.2298486795997703, 2.3166945680443973, 3.0684667441431626, 3.1048613399696188, 2.8558630834656076, 1.7994432383543486, 1.9598193463246933, 1.5271454836210756, 1.0554573719342841, 0.9575689336846319 ], "peak_layer": 26, "peak_normalized": 3.1048613399696188, "population_cv": 0.9076868667660319, "rest_mean": 3.232805502867214e-06, "spike_contrast": 2.8405813003480804, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.183046859106981e-06, "top_five_layers": [ 26, 25, 27, 24, 23 ], "top_five_spike_overlap": 3 }, "values": [ 3.001524873980088e-06, 9.671265388533357e-07, 7.424357022500772e-07, 1.4302667068477604e-06, 1.0998807056239457e-06, 7.6607574328591e-07, 9.134960237133782e-07, 1.471143832532107e-06, 2.0363925159472274e-06, 1.6285648598568514e-06, 1.5240575521602295e-06, 1.6496099988216884e-06, 1.6730530205677496e-06, 1.927663561218651e-06, 2.262119778606575e-06, 2.1471046238730196e-06, 1.5451632862095721e-06, 2.032486463576788e-06, 1.8965492927236483e-06, 1.3752152199231205e-06, 5.715261067962274e-06, 8.502260243403725e-06, 9.281813618144952e-06, 9.643312296248041e-06, 1.277258706977591e-05, 1.2924080692755524e-05, 1.188761780213099e-05, 7.490237749152584e-06, 8.157808224495966e-06, 6.356789981509792e-06, 4.393373728817096e-06, 3.985910097981105e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 1.4861082831885142e-05, "first_to_last_ratio": 0.1969828870720472, "imbalance_abs_log_ratio": 1.6246384216719445, "last_quartile_mean": 7.544352229160722e-05, "mean": 4.024312724482115e-05, "normalized": [ 0.8186900858401205, 0.2924794451740521, 0.14475745634458692, 0.3028320534704602, 0.27849298277360224, 0.22384342400891705, 0.29017017710793486, 0.6029944140644838, 0.6526707522867795, 0.48401173865786107, 0.44195720545378725, 0.46415004448919595, 0.5116229730484052, 0.5221884733509087, 0.5817899634573739, 0.5908748473174855, 0.42739846583695235, 0.5472113748673457, 0.49795092185695733, 0.4744816799850036, 1.3910671942279926, 1.9747247765388387, 2.196275294882107, 2.2898176015603404, 3.138565507461054, 2.911183827350832, 2.442016492317985, 1.6445651638859564, 1.7270745118483022, 1.3905093358120204, 0.9095172284406151, 0.8341145862817433 ], "peak_layer": 25, "peak_normalized": 3.138565507461054, "population_cv": 0.8296567980586155, "rest_mean": 3.1314443664019496e-05, "spike_contrast": 2.824831235395344, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 8.84580185811501e-05, "top_five_layers": [ 25, 26, 27, 24, 23 ], "top_five_spike_overlap": 3 }, "values": [ 3.294664929853752e-05, 1.1770287528634071e-05, 5.825492735311855e-06, 1.2186908861622214e-05, 1.120742854254786e-05, 9.008159395307302e-06, 1.1677355360006914e-05, 2.4266380933113396e-05, 2.6265512133250013e-05, 1.9478145986795425e-05, 1.7785740055842325e-05, 1.8678849301068112e-05, 2.0589308405760676e-05, 2.101449717883952e-05, 2.3413047529174946e-05, 2.377865166636184e-05, 1.719985084491782e-05, 2.202149698860012e-05, 2.003910230996553e-05, 1.909462662297301e-05, 5.598089410341345e-05, 7.94691004557535e-05, 8.838498615659773e-05, 9.214942110702395e-05, 0.00012630569108296186, 0.00011715514119714499, 9.827438043430448e-05, 6.61824451526627e-05, 6.95028793415986e-05, 5.5958444136194885e-05, 3.6601817555492744e-05, 3.3567379432497546e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 9.639708309805428e-06, "first_to_last_ratio": 0.23374050875262048, "imbalance_abs_log_ratio": 1.4535437159045805, "last_quartile_mean": 4.124106840208697e-05, "mean": 2.387599818121089e-05, "normalized": [ 0.8918596537309882, 0.31293373911480943, 0.13153676689798244, 0.29724000229868747, 0.32342785973688676, 0.2579647870849305, 0.3165043136506386, 0.6984571676925886, 0.7521418145225385, 0.547986217455444, 0.49867153377407125, 0.553626029728158, 0.6108020759858102, 0.6119949784087604, 0.6632660891064417, 0.693211154923274, 0.487697562857515, 0.6385011742537995, 0.5581194885228972, 0.5399621240338827, 1.4036275922318706, 1.933773298000513, 2.196390560713897, 2.261885002478936, 3.161583620439522, 2.6751838665470697, 2.1428234671116013, 1.4982028588451668, 1.5150639380661717, 1.2523755187004029, 0.8124804463838124, 0.7607052967009326 ], "peak_layer": 25, "peak_normalized": 3.161583620439522, "population_cv": 0.76724147785331, "rest_mean": 1.8608015563114788e-05, "spike_contrast": 2.8118583704672857, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.232310431892984e-05, "top_five_layers": [ 25, 26, 24, 23, 27 ], "top_five_spike_overlap": 3 }, "values": [ 2.1294039470376447e-05, 7.471605385944713e-06, 3.1405716072185896e-06, 7.0969017542665824e-06, 7.722162990830839e-06, 6.159166787256254e-06, 7.5568564170680474e-06, 1.6676362065481953e-05, 1.7958136595552787e-05, 1.3083717931294814e-05, 1.190628063341137e-05, 1.3218374078860506e-05, 1.458350925531704e-05, 1.461199099139776e-05, 1.5836139937164262e-05, 1.655110827414319e-05, 1.1644266123767011e-05, 1.5244852875184733e-05, 1.3325659892871045e-05, 1.289213469135575e-05, 3.351300983922556e-05, 4.617076774593443e-05, 5.244101703283377e-05, 5.4004762205295265e-05, 7.548596477136016e-05, 6.387268513208255e-05, 5.116204920341261e-05, 3.577108873287216e-05, 3.617366382968612e-05, 2.9901715606683865e-05, 1.9398781660129316e-05, 1.8162598280468956e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 4.471873376132862e-05, "first_to_last_ratio": 0.16588852948022528, "imbalance_abs_log_ratio": 1.7964392253441201, "last_quartile_mean": 0.0002695709817999159, "mean": 0.00013535790498053757, "normalized": [ 0.7401824868370855, 0.26324737157155653, 0.1379665150772088, 0.2887861039317782, 0.237551813451196, 0.19513754079218326, 0.27147014798285596, 0.5086500949015272, 0.5849553550123471, 0.4385958800254328, 0.39932950486729873, 0.4060035899417133, 0.43892200235165824, 0.4611940554720237, 0.5319984000765631, 0.517999329112131, 0.39525956059003536, 0.4770976120171656, 0.45050318830604497, 0.42904324266954524, 1.4032390592771333, 1.9815521512716219, 2.163713643776502, 2.3452636367621564, 3.1293318035380224, 3.06054993338943, 2.7669099713935896, 1.8673427902920254, 1.8460856565125965, 1.449173337860182, 0.9427084291801343, 0.8702357917592554 ], "peak_layer": 25, "peak_normalized": 3.1293318035380224, "population_cv": 0.8885924805056137, "rest_mean": 0.000105162562855791, "spike_contrast": 2.837632940378044, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0002984127524541691, "top_five_layers": [ 25, 26, 27, 24, 23 ], "top_five_spike_overlap": 3 }, "values": [ 0.00010018955072155222, 3.563261270755902e-05, 1.8674858438316733e-05, 3.9089482015697286e-05, 3.215451579308137e-05, 2.6413408704684116e-05, 3.6745630495715886e-05, 6.884981121402234e-05, 7.917833136161789e-05, 5.936741945333779e-05, 5.405240517575294e-05, 5.4955795349087566e-05, 5.941156268818304e-05, 6.242626113817096e-05, 7.201018888736144e-05, 7.011530396994203e-05, 5.350150604499504e-05, 6.457893323386088e-05, 6.097916775615886e-05, 5.8074394473806024e-05, 0.00018993949925061315, 0.000268218747805804, 0.00029287574579939246, 0.00031744997249916196, 0.0004235797969158739, 0.00041426962707191706, 0.00037452313699759543, 0.00025275960797443986, 0.00024988228688016534, 0.000196157066966407, 0.00012760303798131645, 0.00011779329361161217 ] } } } }, "selector": { "expected_uniform_count": 8, "identities_unique": true, "identity_order_sha256": "97fd9721a263024f6a746638f32e0ad9a988cd4c111c00e9e178c1022e5c9d5e", "passed": true, "uniform_count": 8, "uniform_indices": [ 40, 41, 42, 43, 44, 45, 46, 47 ], "visit_count": 65, "visits": [ { "branch": "attention", "coefficient": "detached_learned", "group": 1, "index": 0, "kind": "depth", "layer": 1, "offset": 1, "sources": 1, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 1, "index": 1, "kind": "depth", "layer": 1, "offset": 1, "sources": 2, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 1, "index": 2, "kind": "depth", "layer": 2, "offset": 2, "sources": 2, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 1, "index": 3, "kind": "depth", "layer": 2, "offset": 2, "sources": 2, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 1, "index": 4, "kind": "depth", "layer": 3, "offset": 3, "sources": 2, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 1, "index": 5, "kind": "depth", "layer": 3, "offset": 3, "sources": 2, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 1, "index": 6, "kind": "depth", "layer": 4, "offset": 4, "sources": 2, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 1, "index": 7, "kind": "depth", "layer": 4, "offset": 4, "sources": 2, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 2, "index": 8, "kind": "depth", "layer": 5, "offset": 1, "sources": 2, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 2, "index": 9, "kind": "depth", "layer": 5, "offset": 1, "sources": 3, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 2, "index": 10, "kind": "depth", "layer": 6, "offset": 2, "sources": 3, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 2, "index": 11, "kind": "depth", "layer": 6, "offset": 2, "sources": 3, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 2, "index": 12, "kind": "depth", "layer": 7, "offset": 3, "sources": 3, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 2, "index": 13, "kind": "depth", "layer": 7, "offset": 3, "sources": 3, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 2, "index": 14, "kind": "depth", "layer": 8, "offset": 4, "sources": 3, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 2, "index": 15, "kind": "depth", "layer": 8, "offset": 4, "sources": 3, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 3, "index": 16, "kind": "depth", "layer": 9, "offset": 1, "sources": 3, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 3, "index": 17, "kind": "depth", "layer": 9, "offset": 1, "sources": 4, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 3, "index": 18, "kind": "depth", "layer": 10, "offset": 2, "sources": 4, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 3, "index": 19, "kind": "depth", "layer": 10, "offset": 2, "sources": 4, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 3, "index": 20, "kind": "depth", "layer": 11, "offset": 3, "sources": 4, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 3, "index": 21, "kind": "depth", "layer": 11, "offset": 3, "sources": 4, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 3, "index": 22, "kind": "depth", "layer": 12, "offset": 4, "sources": 4, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 3, "index": 23, "kind": "depth", "layer": 12, "offset": 4, "sources": 4, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 4, "index": 24, "kind": "depth", "layer": 13, "offset": 1, "sources": 4, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 4, "index": 25, "kind": "depth", "layer": 13, "offset": 1, "sources": 5, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 4, "index": 26, "kind": "depth", "layer": 14, "offset": 2, "sources": 5, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 4, "index": 27, "kind": "depth", "layer": 14, "offset": 2, "sources": 5, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 4, "index": 28, "kind": "depth", "layer": 15, "offset": 3, "sources": 5, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 4, "index": 29, "kind": "depth", "layer": 15, "offset": 3, "sources": 5, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 4, "index": 30, "kind": "depth", "layer": 16, "offset": 4, "sources": 5, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 4, "index": 31, "kind": "depth", "layer": 16, "offset": 4, "sources": 5, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 5, "index": 32, "kind": "depth", "layer": 17, "offset": 1, "sources": 5, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 5, "index": 33, "kind": "depth", "layer": 17, "offset": 1, "sources": 6, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 5, "index": 34, "kind": "depth", "layer": 18, "offset": 2, "sources": 6, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 5, "index": 35, "kind": "depth", "layer": 18, "offset": 2, "sources": 6, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 5, "index": 36, "kind": "depth", "layer": 19, "offset": 3, "sources": 6, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 5, "index": 37, "kind": "depth", "layer": 19, "offset": 3, "sources": 6, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 5, "index": 38, "kind": "depth", "layer": 20, "offset": 4, "sources": 6, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 5, "index": 39, "kind": "depth", "layer": 20, "offset": 4, "sources": 6, "use_uniform": false }, { "branch": "attention", "coefficient": "uniform", "group": 6, "index": 40, "kind": "depth", "layer": 21, "offset": 1, "sources": 6, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 6, "index": 41, "kind": "depth", "layer": 21, "offset": 1, "sources": 7, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 6, "index": 42, "kind": "depth", "layer": 22, "offset": 2, "sources": 7, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 6, "index": 43, "kind": "depth", "layer": 22, "offset": 2, "sources": 7, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 6, "index": 44, "kind": "depth", "layer": 23, "offset": 3, "sources": 7, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 6, "index": 45, "kind": "depth", "layer": 23, "offset": 3, "sources": 7, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 6, "index": 46, "kind": "depth", "layer": 24, "offset": 4, "sources": 7, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 6, "index": 47, "kind": "depth", "layer": 24, "offset": 4, "sources": 7, "use_uniform": true }, { "branch": "attention", "coefficient": "detached_learned", "group": 7, "index": 48, "kind": "depth", "layer": 25, "offset": 1, "sources": 7, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 7, "index": 49, "kind": "depth", "layer": 25, "offset": 1, "sources": 8, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 7, "index": 50, "kind": "depth", "layer": 26, "offset": 2, "sources": 8, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 7, "index": 51, "kind": "depth", "layer": 26, "offset": 2, "sources": 8, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 7, "index": 52, "kind": "depth", "layer": 27, "offset": 3, "sources": 8, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 7, "index": 53, "kind": "depth", "layer": 27, "offset": 3, "sources": 8, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 7, "index": 54, "kind": "depth", "layer": 28, "offset": 4, "sources": 8, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 7, "index": 55, "kind": "depth", "layer": 28, "offset": 4, "sources": 8, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 8, "index": 56, "kind": "depth", "layer": 29, "offset": 1, "sources": 8, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 8, "index": 57, "kind": "depth", "layer": 29, "offset": 1, "sources": 9, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 8, "index": 58, "kind": "depth", "layer": 30, "offset": 2, "sources": 9, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 8, "index": 59, "kind": "depth", "layer": 30, "offset": 2, "sources": 9, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 8, "index": 60, "kind": "depth", "layer": 31, "offset": 3, "sources": 9, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 8, "index": 61, "kind": "depth", "layer": 31, "offset": 3, "sources": 9, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 8, "index": 62, "kind": "depth", "layer": 32, "offset": 4, "sources": 9, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 8, "index": 63, "kind": "depth", "layer": 32, "offset": 4, "sources": 9, "use_uniform": false }, { "branch": "output", "coefficient": "detached_learned", "group": null, "index": 64, "kind": "output", "layer": null, "offset": null, "sources": 9, "use_uniform": false } ], "visits_sha256": "a2084e8fee434b0287d854d9862274a543396a9504e35b643f6497973bcbbd04" } }, "uniform_group_7_attention_only": { "bits_per_byte": 1.8740129107275643, "forward": { "activation_sha256": { "attention_branch_output": "5e6ecbcd78543fc42d4098a51169dca446d404ce384015643e84d2c78433c949", "mlp_branch_output": "c0b8018c349b1bd09e5d17858eb312a2323da0e92de9651e2bb08ae4adf115c7", "post_attention_state": "90a633f12d984935028fe5d7212be48e7da7b4a0806751939a551938d16dc540", "post_mlp_state": "89f2226456bfe96e245d2aad0f699d32c676f9f53099789cd870253d002402f5", "pre_attention_input": "ee695e78937de01b7128d3866b8fc5dd328ebdf2c159be37918d4f02952d5248", "pre_mlp_input": "e53fa425d89dfeb0e9395eca9bcd73dc0a55314abf45c5b10b43e2cbfa724345" }, "logits_sha256": "bdacba5675f0579cec629bbf01b36c8660ad576cfdf9fef1bea8259eaeac9ec2", "loss_nats": 1.2989667654037476, "mixer_summary_sha256": "eb3ac0f07de9b755d6e67a510ab68d09d9ad666bb7b5d020ba7d8c5055c399b2" }, "loss_nats": 1.2989667654037476, "loss_scale": 1.0, "mixers": [ { "branch": "attention", "entropy_mean": -0.0, "group": 1, "latest_source_mass": 1.0, "layer": 1, "max_source_mass": 1.0, "mixer_index": 0, "normalized_entropy": 1.0, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 1.0, "median": 1.0, "p05": 1.0, "p95": 1.0 } ], "sources": 1 }, { "branch": "mlp", "entropy_mean": 0.22886337339878082, "group": 1, "latest_source_mass": 0.06350618600845337, "layer": 1, "max_source_mass": 0.9364938139915466, "mixer_index": 1, "normalized_entropy": 0.3301800538435402, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.9364938139915466, "median": 0.9452826976776123, "p05": 0.8725647330284119, "p95": 0.9712384939193726 }, { "label": "current_group_partial", "mean": 0.06350618600845337, "median": 0.0547172985970974, "p05": 0.028761589899659157, "p95": 0.12743529677391052 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.603729784488678, "group": 1, "latest_source_mass": 0.4133155345916748, "layer": 2, "max_source_mass": 0.5866844654083252, "mixer_index": 2, "normalized_entropy": 0.8709979661187783, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.5866844654083252, "median": 0.5734395384788513, "p05": 0.29813364148139954, "p95": 0.8843642473220825 }, { "label": "current_group_partial", "mean": 0.4133155345916748, "median": 0.4265604615211487, "p05": 0.11563576757907867, "p95": 0.7018663287162781 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.3967982828617096, "group": 1, "latest_source_mass": 0.14110636711120605, "layer": 2, "max_source_mass": 0.858893632888794, "mixer_index": 3, "normalized_entropy": 0.5724589149178446, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.858893632888794, "median": 0.8705973029136658, "p05": 0.7599387168884277, "p95": 0.9270997643470764 }, { "label": "current_group_partial", "mean": 0.14110636711120605, "median": 0.12940272688865662, "p05": 0.07290027290582657, "p95": 0.2400612235069275 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6852496862411499, "group": 1, "latest_source_mass": 0.518975019454956, "layer": 3, "max_source_mass": 0.518975019454956, "mixer_index": 4, "normalized_entropy": 0.9886063241108252, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.48102498054504395, "median": 0.48102718591690063, "p05": 0.38786420226097107, "p95": 0.580143928527832 }, { "label": "current_group_partial", "mean": 0.518975019454956, "median": 0.5189728140830994, "p05": 0.41985607147216797, "p95": 0.6121357679367065 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.5890734195709229, "group": 1, "latest_source_mass": 0.3123423159122467, "layer": 3, "max_source_mass": 0.6876576542854309, "mixer_index": 5, "normalized_entropy": 0.8498533011344741, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.6876576542854309, "median": 0.6909762620925903, "p05": 0.48803940415382385, "p95": 0.8900076746940613 }, { "label": "current_group_partial", "mean": 0.3123423159122467, "median": 0.3090237081050873, "p05": 0.10999235510826111, "p95": 0.5119606256484985 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6873323321342468, "group": 1, "latest_source_mass": 0.4831485152244568, "layer": 4, "max_source_mass": 0.5168514251708984, "mixer_index": 6, "normalized_entropy": 0.9916109470127238, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.5168514251708984, "median": 0.5219584703445435, "p05": 0.4251725673675537, "p95": 0.5947866439819336 }, { "label": "current_group_partial", "mean": 0.4831485152244568, "median": 0.47804147005081177, "p05": 0.405213326215744, "p95": 0.5748274326324463 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.487301766872406, "group": 1, "latest_source_mass": 0.2060384750366211, "layer": 4, "max_source_mass": 0.7939615249633789, "mixer_index": 7, "normalized_entropy": 0.7030278424832499, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.7939615249633789, "median": 0.7953404188156128, "p05": 0.6590759754180908, "p95": 0.9329437017440796 }, { "label": "current_group_partial", "mean": 0.2060384750366211, "median": 0.20465964078903198, "p05": 0.06705628335475922, "p95": 0.3409239649772644 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.674697756767273, "group": 2, "latest_source_mass": 0.42952626943588257, "layer": 5, "max_source_mass": 0.5704737305641174, "mixer_index": 8, "normalized_entropy": 0.9733831077870527, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.5704737305641174, "median": 0.5696132779121399, "p05": 0.4695202708244324, "p95": 0.6750621199607849 }, { "label": "completed_group_1", "mean": 0.42952626943588257, "median": 0.4303867816925049, "p05": 0.3249378800392151, "p95": 0.5304797887802124 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.9691112041473389, "group": 2, "latest_source_mass": 0.2681158185005188, "layer": 5, "max_source_mass": 0.5506307482719421, "mixer_index": 9, "normalized_entropy": 0.8821230329784768, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.5506307482719421, "median": 0.543993353843689, "p05": 0.41262108087539673, "p95": 0.7161091566085815 }, { "label": "completed_group_1", "mean": 0.18125343322753906, "median": 0.1794438362121582, "p05": 0.11575750261545181, "p95": 0.25031501054763794 }, { "label": "current_group_partial", "mean": 0.2681158185005188, "median": 0.2664111256599426, "p05": 0.1564716249704361, "p95": 0.3817881643772125 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0745112895965576, "group": 2, "latest_source_mass": 0.412794291973114, "layer": 6, "max_source_mass": 0.412794291973114, "mixer_index": 10, "normalized_entropy": 0.9780623252441762, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.34262996912002563, "median": 0.3412924110889435, "p05": 0.29556700587272644, "p95": 0.39125555753707886 }, { "label": "completed_group_1", "mean": 0.24457570910453796, "median": 0.24382859468460083, "p05": 0.22120657563209534, "p95": 0.2734297513961792 }, { "label": "current_group_partial", "mean": 0.412794291973114, "median": 0.4151499271392822, "p05": 0.37735897302627563, "p95": 0.43749260902404785 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0074045658111572, "group": 2, "latest_source_mass": 0.3775783181190491, "layer": 6, "max_source_mass": 0.4399106502532959, "mixer_index": 11, "normalized_entropy": 0.9169791528842925, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.4399106502532959, "median": 0.44856905937194824, "p05": 0.22063353657722473, "p95": 0.6359403133392334 }, { "label": "completed_group_1", "mean": 0.18251106142997742, "median": 0.18562838435173035, "p05": 0.1196163073182106, "p95": 0.23045453429222107 }, { "label": "current_group_partial", "mean": 0.3775783181190491, "median": 0.3675365447998047, "p05": 0.23817461729049683, "p95": 0.5557451844215393 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.060497522354126, "group": 2, "latest_source_mass": 0.4025024175643921, "layer": 7, "max_source_mass": 0.4025024175643921, "mixer_index": 12, "normalized_entropy": 0.9653064445872968, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.3786153197288513, "median": 0.3907395303249359, "p05": 0.28039106726646423, "p95": 0.4436405599117279 }, { "label": "completed_group_1", "mean": 0.2188822478055954, "median": 0.21226292848587036, "p05": 0.1885872334241867, "p95": 0.2672030031681061 }, { "label": "current_group_partial", "mean": 0.4025024175643921, "median": 0.39815032482147217, "p05": 0.3533596992492676, "p95": 0.460100919008255 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 0.9941019415855408, "group": 2, "latest_source_mass": 0.4801652431488037, "layer": 7, "max_source_mass": 0.4801652431488037, "mixer_index": 13, "normalized_entropy": 0.9048705824970601, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.32296955585479736, "median": 0.2803739905357361, "p05": 0.1784922033548355, "p95": 0.6440245509147644 }, { "label": "completed_group_1", "mean": 0.19686520099639893, "median": 0.2053825855255127, "p05": 0.09813719242811203, "p95": 0.2504696547985077 }, { "label": "current_group_partial", "mean": 0.4801652431488037, "median": 0.5057140588760376, "p05": 0.2529003322124481, "p95": 0.598870575428009 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0503522157669067, "group": 2, "latest_source_mass": 0.4001518487930298, "layer": 8, "max_source_mass": 0.40130966901779175, "mixer_index": 14, "normalized_entropy": 0.9560717885654542, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.40130966901779175, "median": 0.396759569644928, "p05": 0.3479611873626709, "p95": 0.4733521044254303 }, { "label": "completed_group_1", "mean": 0.19853851199150085, "median": 0.19769755005836487, "p05": 0.17136667668819427, "p95": 0.22788912057876587 }, { "label": "current_group_partial", "mean": 0.4001518487930298, "median": 0.40291696786880493, "p05": 0.3506356179714203, "p95": 0.43941178917884827 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0129402875900269, "group": 2, "latest_source_mass": 0.42161279916763306, "layer": 8, "max_source_mass": 0.42161279916763306, "mixer_index": 15, "normalized_entropy": 0.9220179839951123, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.4093267023563385, "median": 0.4046899676322937, "p05": 0.2895161807537079, "p95": 0.5762131810188293 }, { "label": "completed_group_1", "mean": 0.16906049847602844, "median": 0.16888955235481262, "p05": 0.11233551055192947, "p95": 0.22477403283119202 }, { "label": "current_group_partial", "mean": 0.42161279916763306, "median": 0.4227708578109741, "p05": 0.3131049573421478, "p95": 0.519075334072113 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0774019956588745, "group": 3, "latest_source_mass": 0.3560434877872467, "layer": 9, "max_source_mass": 0.38510820269584656, "mixer_index": 16, "normalized_entropy": 0.9806935592947451, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.38510820269584656, "median": 0.3892287313938141, "p05": 0.2934730052947998, "p95": 0.4718902111053467 }, { "label": "completed_group_1", "mean": 0.25884830951690674, "median": 0.25734007358551025, "p05": 0.22217319905757904, "p95": 0.29551801085472107 }, { "label": "completed_group_2", "mean": 0.3560434877872467, "median": 0.35412120819091797, "p05": 0.29444968700408936, "p95": 0.41908568143844604 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.2740377187728882, "group": 3, "latest_source_mass": 0.3633861243724823, "layer": 9, "max_source_mass": 0.3633861243724823, "mixer_index": 17, "normalized_entropy": 0.9190239493895668, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.29550763964653015, "median": 0.30360478162765503, "p05": 0.16904757916927338, "p95": 0.45146965980529785 }, { "label": "completed_group_1", "mean": 0.09541244804859161, "median": 0.09486871212720871, "p05": 0.061360105872154236, "p95": 0.13037073612213135 }, { "label": "completed_group_2", "mean": 0.24569380283355713, "median": 0.24333015084266663, "p05": 0.1698218435049057, "p95": 0.31589236855506897 }, { "label": "current_group_partial", "mean": 0.3633861243724823, "median": 0.36472851037979126, "p05": 0.2795379161834717, "p95": 0.4399876892566681 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.360302448272705, "group": 3, "latest_source_mass": 0.28569501638412476, "layer": 10, "max_source_mass": 0.3109527826309204, "mixer_index": 18, "normalized_entropy": 0.9812507981160736, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.3109527826309204, "median": 0.32172977924346924, "p05": 0.23520725965499878, "p95": 0.37052470445632935 }, { "label": "completed_group_1", "mean": 0.20984819531440735, "median": 0.20958667993545532, "p05": 0.1891367882490158, "p95": 0.23184512555599213 }, { "label": "completed_group_2", "mean": 0.19350400567054749, "median": 0.19153150916099548, "p05": 0.16607457399368286, "p95": 0.22877082228660583 }, { "label": "current_group_partial", "mean": 0.28569501638412476, "median": 0.28060320019721985, "p05": 0.23824527859687805, "p95": 0.3439165949821472 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.2722704410552979, "group": 3, "latest_source_mass": 0.29912811517715454, "layer": 10, "max_source_mass": 0.35036465525627136, "mixer_index": 19, "normalized_entropy": 0.9177491279900463, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.35036465525627136, "median": 0.36210548877716064, "p05": 0.12807878851890564, "p95": 0.5797606706619263 }, { "label": "completed_group_1", "mean": 0.12631990015506744, "median": 0.1264108121395111, "p05": 0.06977735459804535, "p95": 0.1755298376083374 }, { "label": "completed_group_2", "mean": 0.22418735921382904, "median": 0.22563719749450684, "p05": 0.1176326721906662, "p95": 0.32001155614852905 }, { "label": "current_group_partial", "mean": 0.29912811517715454, "median": 0.28589946031570435, "p05": 0.193109393119812, "p95": 0.44119665026664734 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3558036088943481, "group": 3, "latest_source_mass": 0.314197838306427, "layer": 11, "max_source_mass": 0.314197838306427, "mixer_index": 20, "normalized_entropy": 0.978005571485618, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.298976868391037, "median": 0.2961939573287964, "p05": 0.24950163066387177, "p95": 0.35835936665534973 }, { "label": "completed_group_1", "mean": 0.1941717267036438, "median": 0.1918032318353653, "p05": 0.16809208691120148, "p95": 0.22895419597625732 }, { "label": "completed_group_2", "mean": 0.1926535815000534, "median": 0.19306811690330505, "p05": 0.16983944177627563, "p95": 0.2155935913324356 }, { "label": "current_group_partial", "mean": 0.314197838306427, "median": 0.31541913747787476, "p05": 0.2673022747039795, "p95": 0.35941022634506226 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.2696614265441895, "group": 3, "latest_source_mass": 0.2862822115421295, "layer": 11, "max_source_mass": 0.3667730689048767, "mixer_index": 21, "normalized_entropy": 0.9158671218416545, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.3667730689048767, "median": 0.4067659378051758, "p05": 0.10593366622924805, "p95": 0.5672274827957153 }, { "label": "completed_group_1", "mean": 0.15310388803482056, "median": 0.15050218999385834, "p05": 0.08651046454906464, "p95": 0.22459912300109863 }, { "label": "completed_group_2", "mean": 0.19384080171585083, "median": 0.18732509016990662, "p05": 0.1267009675502777, "p95": 0.281107634305954 }, { "label": "current_group_partial", "mean": 0.2862822115421295, "median": 0.26271265745162964, "p05": 0.17024393379688263, "p95": 0.475044310092926 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3606936931610107, "group": 3, "latest_source_mass": 0.3028184771537781, "layer": 12, "max_source_mass": 0.3028184771537781, "mixer_index": 22, "normalized_entropy": 0.9815330216461395, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.298227459192276, "median": 0.2954302132129669, "p05": 0.25287362933158875, "p95": 0.35405808687210083 }, { "label": "completed_group_1", "mean": 0.18611131608486176, "median": 0.1850002110004425, "p05": 0.15901757776737213, "p95": 0.21725836396217346 }, { "label": "completed_group_2", "mean": 0.21284276247024536, "median": 0.21402311325073242, "p05": 0.18819917738437653, "p95": 0.23418807983398438 }, { "label": "current_group_partial", "mean": 0.3028184771537781, "median": 0.3041122555732727, "p05": 0.25661808252334595, "p95": 0.34458184242248535 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.269316554069519, "group": 3, "latest_source_mass": 0.2653314769268036, "layer": 12, "max_source_mass": 0.4081852436065674, "mixer_index": 23, "normalized_entropy": 0.9156183489371815, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.4081852436065674, "median": 0.4254342317581177, "p05": 0.20765496790409088, "p95": 0.587238073348999 }, { "label": "completed_group_1", "mean": 0.14420127868652344, "median": 0.1438048928976059, "p05": 0.1064920499920845, "p95": 0.1871720254421234 }, { "label": "completed_group_2", "mean": 0.18228203058242798, "median": 0.17507661879062653, "p05": 0.12183455377817154, "p95": 0.26505547761917114 }, { "label": "current_group_partial", "mean": 0.2653314769268036, "median": 0.24924281239509583, "p05": 0.15609969198703766, "p95": 0.4061311185359955 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3636136054992676, "group": 4, "latest_source_mass": 0.3136237859725952, "layer": 13, "max_source_mass": 0.3136237859725952, "mixer_index": 24, "normalized_entropy": 0.9836392931712563, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.27427661418914795, "median": 0.27403637766838074, "p05": 0.22928935289382935, "p95": 0.319773405790329 }, { "label": "completed_group_1", "mean": 0.21341991424560547, "median": 0.20697325468063354, "p05": 0.18552586436271667, "p95": 0.26688331365585327 }, { "label": "completed_group_2", "mean": 0.19867971539497375, "median": 0.197545126080513, "p05": 0.1794913411140442, "p95": 0.222100168466568 }, { "label": "completed_group_3", "mean": 0.3136237859725952, "median": 0.31875866651535034, "p05": 0.24156072735786438, "p95": 0.3695686459541321 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.570773959159851, "group": 4, "latest_source_mass": 0.2294243425130844, "layer": 13, "max_source_mass": 0.2658740282058716, "mixer_index": 25, "normalized_entropy": 0.9759767351225285, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.2658740282058716, "median": 0.2747163474559784, "p05": 0.17927880585193634, "p95": 0.3512483835220337 }, { "label": "completed_group_1", "mean": 0.13419361412525177, "median": 0.13374894857406616, "p05": 0.11072897166013718, "p95": 0.1586270034313202 }, { "label": "completed_group_2", "mean": 0.1907704919576645, "median": 0.19074805080890656, "p05": 0.14699944853782654, "p95": 0.23316305875778198 }, { "label": "completed_group_3", "mean": 0.17973749339580536, "median": 0.17616114020347595, "p05": 0.1363743096590042, "p95": 0.23202598094940186 }, { "label": "current_group_partial", "mean": 0.2294243425130844, "median": 0.22995705902576447, "p05": 0.17975632846355438, "p95": 0.2789157032966614 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.5560038089752197, "group": 4, "latest_source_mass": 0.32468360662460327, "layer": 14, "max_source_mass": 0.32468360662460327, "mixer_index": 26, "normalized_entropy": 0.9667995248241249, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.20477107167243958, "median": 0.206403911113739, "p05": 0.16619175672531128, "p95": 0.2431035190820694 }, { "label": "completed_group_1", "mean": 0.15721669793128967, "median": 0.15658551454544067, "p05": 0.13878680765628815, "p95": 0.17778633534908295 }, { "label": "completed_group_2", "mean": 0.13811248540878296, "median": 0.13743364810943604, "p05": 0.12352626770734787, "p95": 0.15553249418735504 }, { "label": "completed_group_3", "mean": 0.1752161681652069, "median": 0.17309701442718506, "p05": 0.15223677456378937, "p95": 0.20474471151828766 }, { "label": "current_group_partial", "mean": 0.32468360662460327, "median": 0.32497870922088623, "p05": 0.2946181893348694, "p95": 0.35203880071640015 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.5659973621368408, "group": 4, "latest_source_mass": 0.26703086495399475, "layer": 14, "max_source_mass": 0.26703086495399475, "mixer_index": 27, "normalized_entropy": 0.9730088685238187, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.20531430840492249, "median": 0.20229148864746094, "p05": 0.10698921978473663, "p95": 0.3393132984638214 }, { "label": "completed_group_1", "mean": 0.15986156463623047, "median": 0.15952220559120178, "p05": 0.11969029903411865, "p95": 0.20180393755435944 }, { "label": "completed_group_2", "mean": 0.21557100117206573, "median": 0.21654027700424194, "p05": 0.1673085242509842, "p95": 0.2610440254211426 }, { "label": "completed_group_3", "mean": 0.15222224593162537, "median": 0.1510905921459198, "p05": 0.11800260841846466, "p95": 0.18893754482269287 }, { "label": "current_group_partial", "mean": 0.26703086495399475, "median": 0.258539617061615, "p05": 0.19330690801143646, "p95": 0.36521556973457336 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.5552496910095215, "group": 4, "latest_source_mass": 0.31774693727493286, "layer": 15, "max_source_mass": 0.31774693727493286, "mixer_index": 28, "normalized_entropy": 0.9663309649872576, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.2162160575389862, "median": 0.2145705670118332, "p05": 0.17727568745613098, "p95": 0.26878347992897034 }, { "label": "completed_group_1", "mean": 0.17982497811317444, "median": 0.18024154007434845, "p05": 0.15453973412513733, "p95": 0.2030555009841919 }, { "label": "completed_group_2", "mean": 0.1307673156261444, "median": 0.13107559084892273, "p05": 0.11463287472724915, "p95": 0.14630235731601715 }, { "label": "completed_group_3", "mean": 0.1554446816444397, "median": 0.15516987442970276, "p05": 0.13627474009990692, "p95": 0.1754152774810791 }, { "label": "current_group_partial", "mean": 0.31774693727493286, "median": 0.319039523601532, "p05": 0.27606821060180664, "p95": 0.35650834441185 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.5068919658660889, "group": 4, "latest_source_mass": 0.3298327624797821, "layer": 15, "max_source_mass": 0.3298327624797821, "mixer_index": 29, "normalized_entropy": 0.9362846209998111, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.20646697282791138, "median": 0.18775638937950134, "p05": 0.0806753933429718, "p95": 0.4274584949016571 }, { "label": "completed_group_1", "mean": 0.1197461187839508, "median": 0.11989086866378784, "p05": 0.0809398666024208, "p95": 0.15743419528007507 }, { "label": "completed_group_2", "mean": 0.19215428829193115, "median": 0.19620640575885773, "p05": 0.1148109883069992, "p95": 0.2491212636232376 }, { "label": "completed_group_3", "mean": 0.15179988741874695, "median": 0.15212899446487427, "p05": 0.10697861760854721, "p95": 0.19542087614536285 }, { "label": "current_group_partial", "mean": 0.3298327624797821, "median": 0.32837340235710144, "p05": 0.22746729850769043, "p95": 0.4339301884174347 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.5435223579406738, "group": 4, "latest_source_mass": 0.3224271535873413, "layer": 16, "max_source_mass": 0.3224271535873413, "mixer_index": 30, "normalized_entropy": 0.9590443632623663, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.22410975396633148, "median": 0.22760415077209473, "p05": 0.14593973755836487, "p95": 0.30093055963516235 }, { "label": "completed_group_1", "mean": 0.16986176371574402, "median": 0.16963240504264832, "p05": 0.13569067418575287, "p95": 0.20651353895664215 }, { "label": "completed_group_2", "mean": 0.13080403208732605, "median": 0.13075244426727295, "p05": 0.10945598781108856, "p95": 0.15187400579452515 }, { "label": "completed_group_3", "mean": 0.15279728174209595, "median": 0.15036612749099731, "p05": 0.12810495495796204, "p95": 0.18468116223812103 }, { "label": "current_group_partial", "mean": 0.3224271535873413, "median": 0.3164973258972168, "p05": 0.2600978910923004, "p95": 0.40530598163604736 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.4949920177459717, "group": 4, "latest_source_mass": 0.31847020983695984, "layer": 16, "max_source_mass": 0.31847020983695984, "mixer_index": 31, "normalized_entropy": 0.9288907675133354, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.27174079418182373, "median": 0.2585451304912567, "p05": 0.13639721274375916, "p95": 0.4698348045349121 }, { "label": "completed_group_1", "mean": 0.12178683280944824, "median": 0.11714950203895569, "p05": 0.08594125509262085, "p95": 0.1723237782716751 }, { "label": "completed_group_2", "mean": 0.1529826819896698, "median": 0.15223106741905212, "p05": 0.10371473431587219, "p95": 0.203575000166893 }, { "label": "completed_group_3", "mean": 0.1350194811820984, "median": 0.13363957405090332, "p05": 0.08964549005031586, "p95": 0.18468207120895386 }, { "label": "current_group_partial", "mean": 0.31847020983695984, "median": 0.3185216188430786, "p05": 0.21444711089134216, "p95": 0.4301546812057495 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.544264793395996, "group": 5, "latest_source_mass": 0.2861037850379944, "layer": 17, "max_source_mass": 0.2861037850379944, "mixer_index": 32, "normalized_entropy": 0.9595056643474137, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.24016115069389343, "median": 0.23868846893310547, "p05": 0.18435582518577576, "p95": 0.2983453571796417 }, { "label": "completed_group_1", "mean": 0.20730282366275787, "median": 0.1998077780008316, "p05": 0.16255050897598267, "p95": 0.28107208013534546 }, { "label": "completed_group_2", "mean": 0.09692573547363281, "median": 0.0967404842376709, "p05": 0.08098706603050232, "p95": 0.11221462488174438 }, { "label": "completed_group_3", "mean": 0.1695065200328827, "median": 0.1696585714817047, "p05": 0.136396586894989, "p95": 0.20130634307861328 }, { "label": "completed_group_4", "mean": 0.2861037850379944, "median": 0.28883200883865356, "p05": 0.22401469945907593, "p95": 0.3409370183944702 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.686682939529419, "group": 5, "latest_source_mass": 0.12281867861747742, "layer": 17, "max_source_mass": 0.2885604202747345, "mixer_index": 33, "normalized_entropy": 0.9413556721740635, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.16755425930023193, "median": 0.1327776312828064, "p05": 0.0790192037820816, "p95": 0.3667412996292114 }, { "label": "completed_group_1", "mean": 0.10962115228176117, "median": 0.10525335371494293, "p05": 0.07840172946453094, "p95": 0.15405339002609253 }, { "label": "completed_group_2", "mean": 0.1783454418182373, "median": 0.17896884679794312, "p05": 0.10802385956048965, "p95": 0.24048911035060883 }, { "label": "completed_group_3", "mean": 0.13310004770755768, "median": 0.13033035397529602, "p05": 0.0904831737279892, "p95": 0.17934636771678925 }, { "label": "completed_group_4", "mean": 0.2885604202747345, "median": 0.2831825315952301, "p05": 0.16967125236988068, "p95": 0.4189499616622925 }, { "label": "current_group_partial", "mean": 0.12281867861747742, "median": 0.12062528729438782, "p05": 0.07461707293987274, "p95": 0.1782219111919403 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.7070834636688232, "group": 5, "latest_source_mass": 0.2988377809524536, "layer": 18, "max_source_mass": 0.2988377809524536, "mixer_index": 34, "normalized_entropy": 0.9527414214834803, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.181177020072937, "median": 0.18419909477233887, "p05": 0.13624237477779388, "p95": 0.22192169725894928 }, { "label": "completed_group_1", "mean": 0.14557164907455444, "median": 0.144476979970932, "p05": 0.1293315887451172, "p95": 0.16486899554729462 }, { "label": "completed_group_2", "mean": 0.08318156749010086, "median": 0.0820261612534523, "p05": 0.07088497281074524, "p95": 0.0990293100476265 }, { "label": "completed_group_3", "mean": 0.11424209177494049, "median": 0.11267061531543732, "p05": 0.0979941189289093, "p95": 0.1353992223739624 }, { "label": "completed_group_4", "mean": 0.17698991298675537, "median": 0.17597860097885132, "p05": 0.14837387204170227, "p95": 0.21050144731998444 }, { "label": "current_group_partial", "mean": 0.2988377809524536, "median": 0.2989804744720459, "p05": 0.2721370458602905, "p95": 0.32572072744369507 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.7419235706329346, "group": 5, "latest_source_mass": 0.26016175746917725, "layer": 18, "max_source_mass": 0.26016175746917725, "mixer_index": 35, "normalized_entropy": 0.972186055410333, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1532188355922699, "median": 0.1451343297958374, "p05": 0.11514341086149216, "p95": 0.22103679180145264 }, { "label": "completed_group_1", "mean": 0.11420352011919022, "median": 0.11261284351348877, "p05": 0.0928458571434021, "p95": 0.14123988151550293 }, { "label": "completed_group_2", "mean": 0.16597424447536469, "median": 0.1655328869819641, "p05": 0.13871800899505615, "p95": 0.19611573219299316 }, { "label": "completed_group_3", "mean": 0.1250365972518921, "median": 0.12369821965694427, "p05": 0.10375828295946121, "p95": 0.14912742376327515 }, { "label": "completed_group_4", "mean": 0.18140508234500885, "median": 0.1796584278345108, "p05": 0.14511024951934814, "p95": 0.22286373376846313 }, { "label": "current_group_partial", "mean": 0.26016175746917725, "median": 0.26266711950302124, "p05": 0.1889389455318451, "p95": 0.3234381675720215 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.7127439975738525, "group": 5, "latest_source_mass": 0.2907913327217102, "layer": 19, "max_source_mass": 0.2907913327217102, "mixer_index": 36, "normalized_entropy": 0.9559006256078307, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.18731768429279327, "median": 0.19042012095451355, "p05": 0.13879702985286713, "p95": 0.23792000114917755 }, { "label": "completed_group_1", "mean": 0.16715484857559204, "median": 0.1667211651802063, "p05": 0.14574480056762695, "p95": 0.19026139378547668 }, { "label": "completed_group_2", "mean": 0.08496320247650146, "median": 0.08452782034873962, "p05": 0.07410907745361328, "p95": 0.097386933863163 }, { "label": "completed_group_3", "mean": 0.11362075805664062, "median": 0.11248823255300522, "p05": 0.0977015346288681, "p95": 0.13219426572322845 }, { "label": "completed_group_4", "mean": 0.15615218877792358, "median": 0.15540224313735962, "p05": 0.1289147436618805, "p95": 0.18598109483718872 }, { "label": "current_group_partial", "mean": 0.2907913327217102, "median": 0.2900415062904358, "p05": 0.25822532176971436, "p95": 0.32475030422210693 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.6076834201812744, "group": 5, "latest_source_mass": 0.3738824725151062, "layer": 19, "max_source_mass": 0.3738824725151062, "mixer_index": 37, "normalized_entropy": 0.8972652009334232, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.15032559633255005, "median": 0.12544110417366028, "p05": 0.06976518034934998, "p95": 0.35840943455696106 }, { "label": "completed_group_1", "mean": 0.08624927699565887, "median": 0.08602266013622284, "p05": 0.061998721212148666, "p95": 0.11138717085123062 }, { "label": "completed_group_2", "mean": 0.15736976265907288, "median": 0.15802210569381714, "p05": 0.10121746361255646, "p95": 0.20687000453472137 }, { "label": "completed_group_3", "mean": 0.08584291487932205, "median": 0.0827617198228836, "p05": 0.060561805963516235, "p95": 0.11351694166660309 }, { "label": "completed_group_4", "mean": 0.14632999897003174, "median": 0.1426895558834076, "p05": 0.10106660425662994, "p95": 0.20179018378257751 }, { "label": "current_group_partial", "mean": 0.3738824725151062, "median": 0.3799295425415039, "p05": 0.23631010949611664, "p95": 0.4884527325630188 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.709290862083435, "group": 5, "latest_source_mass": 0.2901708781719208, "layer": 20, "max_source_mass": 0.2901708781719208, "mixer_index": 38, "normalized_entropy": 0.9539733939957076, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.18991519510746002, "median": 0.1924753338098526, "p05": 0.14201581478118896, "p95": 0.23709386587142944 }, { "label": "completed_group_1", "mean": 0.17091524600982666, "median": 0.17006167769432068, "p05": 0.15297070145606995, "p95": 0.19153237342834473 }, { "label": "completed_group_2", "mean": 0.07822999358177185, "median": 0.07778562605381012, "p05": 0.06707640737295151, "p95": 0.09089364111423492 }, { "label": "completed_group_3", "mean": 0.116765096783638, "median": 0.11516137421131134, "p05": 0.10162825882434845, "p95": 0.13653048872947693 }, { "label": "completed_group_4", "mean": 0.1540035903453827, "median": 0.15287795662879944, "p05": 0.12593276798725128, "p95": 0.18590238690376282 }, { "label": "current_group_partial", "mean": 0.2901708781719208, "median": 0.29104921221733093, "p05": 0.25643545389175415, "p95": 0.3205650746822357 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.6690887212753296, "group": 5, "latest_source_mass": 0.3342471718788147, "layer": 20, "max_source_mass": 0.3342471718788147, "mixer_index": 39, "normalized_entropy": 0.9315361520005944, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.19029226899147034, "median": 0.19644704461097717, "p05": 0.09456793963909149, "p95": 0.30468082427978516 }, { "label": "completed_group_1", "mean": 0.1100241169333458, "median": 0.10808776319026947, "p05": 0.09035106748342514, "p95": 0.1359245330095291 }, { "label": "completed_group_2", "mean": 0.11471512168645859, "median": 0.11494918912649155, "p05": 0.08799737691879272, "p95": 0.14024731516838074 }, { "label": "completed_group_3", "mean": 0.09999909996986389, "median": 0.09682698547840118, "p05": 0.07921088486909866, "p95": 0.1264391839504242 }, { "label": "completed_group_4", "mean": 0.1507222205400467, "median": 0.1476232260465622, "p05": 0.11397356539964676, "p95": 0.19991227984428406 }, { "label": "current_group_partial", "mean": 0.3342471718788147, "median": 0.33802270889282227, "p05": 0.2548266053199768, "p95": 0.40130624175071716 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.7458693981170654, "group": 6, "latest_source_mass": 0.2361394464969635, "layer": 21, "max_source_mass": 0.2361394464969635, "mixer_index": 40, "normalized_entropy": 0.9743882636597644, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.20528709888458252, "median": 0.2140858769416809, "p05": 0.13168928027153015, "p95": 0.27098023891448975 }, { "label": "completed_group_1", "mean": 0.1722964197397232, "median": 0.17238952219486237, "p05": 0.15053534507751465, "p95": 0.19365043938159943 }, { "label": "completed_group_2", "mean": 0.10435444861650467, "median": 0.10361101478338242, "p05": 0.08888213336467743, "p95": 0.12212536484003067 }, { "label": "completed_group_3", "mean": 0.12075909972190857, "median": 0.1180175170302391, "p05": 0.10358668118715286, "p95": 0.14458929002285004 }, { "label": "completed_group_4", "mean": 0.16116347908973694, "median": 0.15841388702392578, "p05": 0.1337081342935562, "p95": 0.19791515171527863 }, { "label": "completed_group_5", "mean": 0.2361394464969635, "median": 0.23709020018577576, "p05": 0.20446214079856873, "p95": 0.26509594917297363 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.4612125158309937, "group": 6, "latest_source_mass": 0.5543455481529236, "layer": 21, "max_source_mass": 0.5543455481529236, "mixer_index": 41, "normalized_entropy": 0.7509146897354808, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.08934466540813446, "median": 0.07911523431539536, "p05": 0.04488275945186615, "p95": 0.1737232506275177 }, { "label": "completed_group_1", "mean": 0.054379627108573914, "median": 0.05344422906637192, "p05": 0.04221750423312187, "p95": 0.06924191862344742 }, { "label": "completed_group_2", "mean": 0.07567165046930313, "median": 0.07438549399375916, "p05": 0.05811355262994766, "p95": 0.09632256627082825 }, { "label": "completed_group_3", "mean": 0.05758609622716904, "median": 0.05612001568078995, "p05": 0.044792283326387405, "p95": 0.07347827404737473 }, { "label": "completed_group_4", "mean": 0.07884950935840607, "median": 0.07757881283760071, "p05": 0.05758974328637123, "p95": 0.10452556610107422 }, { "label": "completed_group_5", "mean": 0.08982288837432861, "median": 0.08912864327430725, "p05": 0.0663854256272316, "p95": 0.11604885756969452 }, { "label": "current_group_partial", "mean": 0.5543455481529236, "median": 0.5596089363098145, "p05": 0.45455610752105713, "p95": 0.6367484331130981 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.7241814136505127, "group": 6, "latest_source_mass": 0.39484071731567383, "layer": 22, "max_source_mass": 0.39484071731567383, "mixer_index": 42, "normalized_entropy": 0.886053970419732, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.13360010087490082, "median": 0.14072942733764648, "p05": 0.07699814438819885, "p95": 0.17506597936153412 }, { "label": "completed_group_1", "mean": 0.10609637200832367, "median": 0.10586297512054443, "p05": 0.08648421615362167, "p95": 0.12602579593658447 }, { "label": "completed_group_2", "mean": 0.06363911926746368, "median": 0.06324908882379532, "p05": 0.054980404675006866, "p95": 0.07316673547029495 }, { "label": "completed_group_3", "mean": 0.07426095008850098, "median": 0.07362888753414154, "p05": 0.06318790465593338, "p95": 0.08665015548467636 }, { "label": "completed_group_4", "mean": 0.09383076429367065, "median": 0.09294568747282028, "p05": 0.07638302445411682, "p95": 0.11445100605487823 }, { "label": "completed_group_5", "mean": 0.13373197615146637, "median": 0.13327741622924805, "p05": 0.11399447172880173, "p95": 0.15641780197620392 }, { "label": "current_group_partial", "mean": 0.39484071731567383, "median": 0.3879151940345764, "p05": 0.3302172124385834, "p95": 0.4852263331413269 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.3979300260543823, "group": 6, "latest_source_mass": 0.5830056071281433, "layer": 22, "max_source_mass": 0.5830056071281433, "mixer_index": 43, "normalized_entropy": 0.7183939231382495, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.08959861099720001, "median": 0.07544538378715515, "p05": 0.04165133833885193, "p95": 0.20099139213562012 }, { "label": "completed_group_1", "mean": 0.05414731800556183, "median": 0.05336771905422211, "p05": 0.04069708660244942, "p95": 0.07046990841627121 }, { "label": "completed_group_2", "mean": 0.07668881118297577, "median": 0.07607124000787735, "p05": 0.0555989071726799, "p95": 0.09980960935354233 }, { "label": "completed_group_3", "mean": 0.05603286623954773, "median": 0.054573021829128265, "p05": 0.04162490367889404, "p95": 0.0730118378996849 }, { "label": "completed_group_4", "mean": 0.06979228556156158, "median": 0.06884932518005371, "p05": 0.04576495662331581, "p95": 0.09806846082210541 }, { "label": "completed_group_5", "mean": 0.07073447108268738, "median": 0.07023533433675766, "p05": 0.04956750199198723, "p95": 0.09379896521568298 }, { "label": "current_group_partial", "mean": 0.5830056071281433, "median": 0.5862616896629333, "p05": 0.4658474922180176, "p95": 0.6849814057350159 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.7459560632705688, "group": 6, "latest_source_mass": 0.37691548466682434, "layer": 23, "max_source_mass": 0.37691548466682434, "mixer_index": 44, "normalized_entropy": 0.897243926765161, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.14318783581256866, "median": 0.1493408977985382, "p05": 0.08292607218027115, "p95": 0.19464610517024994 }, { "label": "completed_group_1", "mean": 0.1094653457403183, "median": 0.10858799517154694, "p05": 0.09432064741849899, "p95": 0.1279931515455246 }, { "label": "completed_group_2", "mean": 0.06349228322505951, "median": 0.06325754523277283, "p05": 0.05439673364162445, "p95": 0.0737837627530098 }, { "label": "completed_group_3", "mean": 0.07859592139720917, "median": 0.0775163471698761, "p05": 0.06603655964136124, "p95": 0.09357087314128876 }, { "label": "completed_group_4", "mean": 0.09567482024431229, "median": 0.09468592703342438, "p05": 0.07671141624450684, "p95": 0.11774884164333344 }, { "label": "completed_group_5", "mean": 0.13266831636428833, "median": 0.13199329376220703, "p05": 0.11244359612464905, "p95": 0.15504513680934906 }, { "label": "current_group_partial", "mean": 0.37691548466682434, "median": 0.37180638313293457, "p05": 0.31207239627838135, "p95": 0.4595233201980591 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.4055393934249878, "group": 6, "latest_source_mass": 0.5772827863693237, "layer": 23, "max_source_mass": 0.5772827863693237, "mixer_index": 45, "normalized_entropy": 0.7223043644164862, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.10154955089092255, "median": 0.08766824007034302, "p05": 0.04901552200317383, "p95": 0.20287738740444183 }, { "label": "completed_group_1", "mean": 0.05619276314973831, "median": 0.055477917194366455, "p05": 0.041608311235904694, "p95": 0.07371112704277039 }, { "label": "completed_group_2", "mean": 0.07791627198457718, "median": 0.07736755907535553, "p05": 0.055920060724020004, "p95": 0.10158214718103409 }, { "label": "completed_group_3", "mean": 0.053915880620479584, "median": 0.05215863138437271, "p05": 0.03798118978738785, "p95": 0.07156119495630264 }, { "label": "completed_group_4", "mean": 0.06398355215787888, "median": 0.06229078769683838, "p05": 0.040616974234580994, "p95": 0.09270931780338287 }, { "label": "completed_group_5", "mean": 0.06915920227766037, "median": 0.0684593990445137, "p05": 0.046986356377601624, "p95": 0.09408729523420334 }, { "label": "current_group_partial", "mean": 0.5772827863693237, "median": 0.5824689269065857, "p05": 0.4518115520477295, "p95": 0.6926202178001404 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.7853620052337646, "group": 6, "latest_source_mass": 0.3407441973686218, "layer": 24, "max_source_mass": 0.3407441973686218, "mixer_index": 46, "normalized_entropy": 0.9174945750195659, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.15284113585948944, "median": 0.1593906283378601, "p05": 0.08678612112998962, "p95": 0.211342453956604 }, { "label": "completed_group_1", "mean": 0.12206703424453735, "median": 0.12153112143278122, "p05": 0.1060115247964859, "p95": 0.1399766057729721 }, { "label": "completed_group_2", "mean": 0.06508444249629974, "median": 0.06482555717229843, "p05": 0.055350616574287415, "p95": 0.07557765394449234 }, { "label": "completed_group_3", "mean": 0.08347313106060028, "median": 0.08190013468265533, "p05": 0.06966532766819, "p95": 0.10050469636917114 }, { "label": "completed_group_4", "mean": 0.09811970591545105, "median": 0.09664031863212585, "p05": 0.07688035070896149, "p95": 0.12369741499423981 }, { "label": "completed_group_5", "mean": 0.13767032325267792, "median": 0.1373325139284134, "p05": 0.11642646044492722, "p95": 0.1606694608926773 }, { "label": "current_group_partial", "mean": 0.3407441973686218, "median": 0.33805108070373535, "p05": 0.271301805973053, "p95": 0.41781002283096313 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.509842872619629, "group": 6, "latest_source_mass": 0.5227814316749573, "layer": 24, "max_source_mass": 0.5227814316749573, "mixer_index": 47, "normalized_entropy": 0.7759057494780099, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.11788781732320786, "median": 0.10097194463014603, "p05": 0.06265684217214584, "p95": 0.2408325970172882 }, { "label": "completed_group_1", "mean": 0.06822571158409119, "median": 0.0673636943101883, "p05": 0.05209728330373764, "p95": 0.08648915588855743 }, { "label": "completed_group_2", "mean": 0.08556114137172699, "median": 0.08463940769433975, "p05": 0.06234566867351532, "p95": 0.1125786080956459 }, { "label": "completed_group_3", "mean": 0.06067090481519699, "median": 0.058974385261535645, "p05": 0.043267179280519485, "p95": 0.07997901737689972 }, { "label": "completed_group_4", "mean": 0.07228855788707733, "median": 0.06972275674343109, "p05": 0.04689513146877289, "p95": 0.106512151658535 }, { "label": "completed_group_5", "mean": 0.07258446514606476, "median": 0.07082539796829224, "p05": 0.050949547439813614, "p95": 0.09970614314079285 }, { "label": "current_group_partial", "mean": 0.5227814316749573, "median": 0.5290273427963257, "p05": 0.38773614168167114, "p95": 0.6392589807510376 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.839367389678955, "group": 7, "latest_source_mass": 0.29583215713500977, "layer": 25, "max_source_mass": 0.29583215713500977, "mixer_index": 48, "normalized_entropy": 0.9452478525649903, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.15906387567520142, "median": 0.15846745669841766, "p05": 0.09519976377487183, "p95": 0.24517810344696045 }, { "label": "completed_group_1", "mean": 0.13139380514621735, "median": 0.1308426856994629, "p05": 0.11478274315595627, "p95": 0.15000829100608826 }, { "label": "completed_group_2", "mean": 0.07717745751142502, "median": 0.07715500891208649, "p05": 0.06616044044494629, "p95": 0.08791667222976685 }, { "label": "completed_group_3", "mean": 0.09607303142547607, "median": 0.09436552226543427, "p05": 0.08152313530445099, "p95": 0.11384639143943787 }, { "label": "completed_group_4", "mean": 0.1107850894331932, "median": 0.1089654266834259, "p05": 0.08814431726932526, "p95": 0.139452263712883 }, { "label": "completed_group_5", "mean": 0.12967455387115479, "median": 0.1299237608909607, "p05": 0.10832332074642181, "p95": 0.15138982236385345 }, { "label": "completed_group_6", "mean": 0.29583215713500977, "median": 0.29647016525268555, "p05": 0.2276897132396698, "p95": 0.3604946434497833 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.5841834545135498, "group": 7, "latest_source_mass": 0.40767741203308105, "layer": 25, "max_source_mass": 0.40767741203308105, "mixer_index": 49, "normalized_entropy": 0.7618312045616817, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.07896879315376282, "median": 0.06809917092323303, "p05": 0.025932887569069862, "p95": 0.19368338584899902 }, { "label": "completed_group_1", "mean": 0.048752427101135254, "median": 0.04767637699842453, "p05": 0.034443892538547516, "p95": 0.0667153000831604 }, { "label": "completed_group_2", "mean": 0.055707164108753204, "median": 0.0549345538020134, "p05": 0.038696639239788055, "p95": 0.0750277042388916 }, { "label": "completed_group_3", "mean": 0.04139826446771622, "median": 0.03878612071275711, "p05": 0.026673054322600365, "p95": 0.05746251344680786 }, { "label": "completed_group_4", "mean": 0.04449018836021423, "median": 0.04196460545063019, "p05": 0.026162121444940567, "p95": 0.07164917886257172 }, { "label": "completed_group_5", "mean": 0.04285799711942673, "median": 0.04109475016593933, "p05": 0.029393533244729042, "p95": 0.06277710199356079 }, { "label": "completed_group_6", "mean": 0.2801477909088135, "median": 0.2763082981109619, "p05": 0.1455708146095276, "p95": 0.4254714846611023 }, { "label": "current_group_partial", "mean": 0.40767741203308105, "median": 0.4120897948741913, "p05": 0.21399420499801636, "p95": 0.5853031277656555 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.8125314712524414, "group": 7, "latest_source_mass": 0.3700707256793976, "layer": 26, "max_source_mass": 0.3700707256793976, "mixer_index": 50, "normalized_entropy": 0.8716433883436914, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.10889513790607452, "median": 0.11333495378494263, "p05": 0.04850132763385773, "p95": 0.1620434820652008 }, { "label": "completed_group_1", "mean": 0.08656036853790283, "median": 0.08640051633119583, "p05": 0.0650845319032669, "p95": 0.1084962785243988 }, { "label": "completed_group_2", "mean": 0.048123255372047424, "median": 0.04800213500857353, "p05": 0.03576180338859558, "p95": 0.06107594817876816 }, { "label": "completed_group_3", "mean": 0.06561170518398285, "median": 0.06379580497741699, "p05": 0.050968848168849945, "p95": 0.08332888036966324 }, { "label": "completed_group_4", "mean": 0.06917259097099304, "median": 0.06724178791046143, "p05": 0.050207968801259995, "p95": 0.09373342245817184 }, { "label": "completed_group_5", "mean": 0.08286689221858978, "median": 0.08183546364307404, "p05": 0.06535022705793381, "p95": 0.10340933501720428 }, { "label": "completed_group_6", "mean": 0.16869932413101196, "median": 0.16509854793548584, "p05": 0.11059704422950745, "p95": 0.23654375970363617 }, { "label": "current_group_partial", "mean": 0.3700707256793976, "median": 0.3694133758544922, "p05": 0.23441238701343536, "p95": 0.5025027990341187 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.50089693069458, "group": 7, "latest_source_mass": 0.49096518754959106, "layer": 26, "max_source_mass": 0.49096518754959106, "mixer_index": 51, "normalized_entropy": 0.7217788529328457, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.07974998652935028, "median": 0.07402504235506058, "p05": 0.04579866677522659, "p95": 0.1359037309885025 }, { "label": "completed_group_1", "mean": 0.04166398197412491, "median": 0.04038450866937637, "p05": 0.027992289513349533, "p95": 0.05969971418380737 }, { "label": "completed_group_2", "mean": 0.04607272148132324, "median": 0.044438350945711136, "p05": 0.030038591474294662, "p95": 0.06808309257030487 }, { "label": "completed_group_3", "mean": 0.03586764633655548, "median": 0.03317485377192497, "p05": 0.02203454077243805, "p95": 0.05572832003235817 }, { "label": "completed_group_4", "mean": 0.04223340004682541, "median": 0.03930673003196716, "p05": 0.02465122565627098, "p95": 0.06968094408512115 }, { "label": "completed_group_5", "mean": 0.04095568507909775, "median": 0.03919431194663048, "p05": 0.026239857077598572, "p95": 0.06058088690042496 }, { "label": "completed_group_6", "mean": 0.22249138355255127, "median": 0.2125401496887207, "p05": 0.10647299885749817, "p95": 0.3711881935596466 }, { "label": "current_group_partial", "mean": 0.49096518754959106, "median": 0.4988659620285034, "p05": 0.2878011167049408, "p95": 0.6676807403564453 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.8059929609298706, "group": 7, "latest_source_mass": 0.37139976024627686, "layer": 27, "max_source_mass": 0.37139976024627686, "mixer_index": 52, "normalized_entropy": 0.8684990295379666, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.10711923241615295, "median": 0.10972198843955994, "p05": 0.04896187782287598, "p95": 0.16201043128967285 }, { "label": "completed_group_1", "mean": 0.07651707530021667, "median": 0.07666032761335373, "p05": 0.05141128972172737, "p95": 0.10152198374271393 }, { "label": "completed_group_2", "mean": 0.058284737169742584, "median": 0.05812416970729828, "p05": 0.039273619651794434, "p95": 0.07745087146759033 }, { "label": "completed_group_3", "mean": 0.06609587371349335, "median": 0.06383530795574188, "p05": 0.04481930285692215, "p95": 0.09336265176534653 }, { "label": "completed_group_4", "mean": 0.07139429450035095, "median": 0.0694514587521553, "p05": 0.0485839769244194, "p95": 0.10034126043319702 }, { "label": "completed_group_5", "mean": 0.07493151724338531, "median": 0.07319007813930511, "p05": 0.051876723766326904, "p95": 0.10402432829141617 }, { "label": "completed_group_6", "mean": 0.17425751686096191, "median": 0.16937679052352905, "p05": 0.10449051856994629, "p95": 0.2624811828136444 }, { "label": "current_group_partial", "mean": 0.37139976024627686, "median": 0.3732905089855194, "p05": 0.23830299079418182, "p95": 0.5014542937278748 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.5622732639312744, "group": 7, "latest_source_mass": 0.4768889844417572, "layer": 27, "max_source_mass": 0.4768889844417572, "mixer_index": 53, "normalized_entropy": 0.7512946301290215, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.08775827288627625, "median": 0.08504067361354828, "p05": 0.0428270623087883, "p95": 0.144285649061203 }, { "label": "completed_group_1", "mean": 0.047765426337718964, "median": 0.04646281152963638, "p05": 0.031814731657505035, "p95": 0.06807165592908859 }, { "label": "completed_group_2", "mean": 0.053928617388010025, "median": 0.052262187004089355, "p05": 0.035437777638435364, "p95": 0.0784272700548172 }, { "label": "completed_group_3", "mean": 0.04293380305171013, "median": 0.03996016085147858, "p05": 0.02556057646870613, "p95": 0.06623870134353638 }, { "label": "completed_group_4", "mean": 0.04174989461898804, "median": 0.03903091698884964, "p05": 0.025013400241732597, "p95": 0.06869364529848099 }, { "label": "completed_group_5", "mean": 0.04734005779027939, "median": 0.04609068110585213, "p05": 0.029884062707424164, "p95": 0.06911227852106094 }, { "label": "completed_group_6", "mean": 0.20163494348526, "median": 0.19222964346408844, "p05": 0.09963390231132507, "p95": 0.3307305574417114 }, { "label": "current_group_partial", "mean": 0.4768889844417572, "median": 0.48383551836013794, "p05": 0.276212215423584, "p95": 0.6487575173377991 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.8757455348968506, "group": 7, "latest_source_mass": 0.3109493851661682, "layer": 28, "max_source_mass": 0.3109493851661682, "mixer_index": 54, "normalized_entropy": 0.9020429270551009, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.11563669890165329, "median": 0.12087491154670715, "p05": 0.05733206868171692, "p95": 0.1688953936100006 }, { "label": "completed_group_1", "mean": 0.08108697831630707, "median": 0.08185897022485733, "p05": 0.05175650864839554, "p95": 0.10753051191568375 }, { "label": "completed_group_2", "mean": 0.07280858606100082, "median": 0.07357072830200195, "p05": 0.04870978742837906, "p95": 0.0948689877986908 }, { "label": "completed_group_3", "mean": 0.07553227245807648, "median": 0.07362619787454605, "p05": 0.056317005306482315, "p95": 0.09756095707416534 }, { "label": "completed_group_4", "mean": 0.07606014609336853, "median": 0.07456053793430328, "p05": 0.05528895929455757, "p95": 0.1022825762629509 }, { "label": "completed_group_5", "mean": 0.06935927271842957, "median": 0.06843917071819305, "p05": 0.04978285729885101, "p95": 0.09167985618114471 }, { "label": "completed_group_6", "mean": 0.19856664538383484, "median": 0.19508783519268036, "p05": 0.1281973123550415, "p95": 0.27796900272369385 }, { "label": "current_group_partial", "mean": 0.3109493851661682, "median": 0.3105033040046692, "p05": 0.18172834813594818, "p95": 0.43916669487953186 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.6924852132797241, "group": 7, "latest_source_mass": 0.3315581679344177, "layer": 28, "max_source_mass": 0.3315581679344177, "mixer_index": 55, "normalized_entropy": 0.8139133413255193, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.08638577908277512, "median": 0.058608733117580414, "p05": 0.026544082909822464, "p95": 0.2740764319896698 }, { "label": "completed_group_1", "mean": 0.059912845492362976, "median": 0.05901291221380234, "p05": 0.03629481792449951, "p95": 0.08707985281944275 }, { "label": "completed_group_2", "mean": 0.06609142571687698, "median": 0.0647939145565033, "p05": 0.038691695779561996, "p95": 0.09703366458415985 }, { "label": "completed_group_3", "mean": 0.056745611131191254, "median": 0.05418796092271805, "p05": 0.032174739986658096, "p95": 0.08370660990476608 }, { "label": "completed_group_4", "mean": 0.06277773529291153, "median": 0.0600711889564991, "p05": 0.03501531109213829, "p95": 0.10094858705997467 }, { "label": "completed_group_5", "mean": 0.05352145433425903, "median": 0.05179818347096443, "p05": 0.032271772623062134, "p95": 0.08084158599376678 }, { "label": "completed_group_6", "mean": 0.2830069661140442, "median": 0.2759752869606018, "p05": 0.12981083989143372, "p95": 0.46204638481140137 }, { "label": "current_group_partial", "mean": 0.3315581679344177, "median": 0.3204721510410309, "p05": 0.13446375727653503, "p95": 0.5667697787284851 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.8980522155761719, "group": 8, "latest_source_mass": 0.20123262703418732, "layer": 29, "max_source_mass": 0.26602888107299805, "mixer_index": 56, "normalized_entropy": 0.9127701729200177, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.11603894829750061, "median": 0.09991352260112762, "p05": 0.030035395175218582, "p95": 0.2941743731498718 }, { "label": "completed_group_1", "mean": 0.08804173022508621, "median": 0.08880250155925751, "p05": 0.05664174258708954, "p95": 0.11779069900512695 }, { "label": "completed_group_2", "mean": 0.07975024729967117, "median": 0.07984748482704163, "p05": 0.05428915470838547, "p95": 0.10499341040849686 }, { "label": "completed_group_3", "mean": 0.09027000516653061, "median": 0.08898856490850449, "p05": 0.06189651042222977, "p95": 0.11996553838253021 }, { "label": "completed_group_4", "mean": 0.08430293202400208, "median": 0.0819794088602066, "p05": 0.05464346706867218, "p95": 0.12159879505634308 }, { "label": "completed_group_5", "mean": 0.07433462142944336, "median": 0.07287181913852692, "p05": 0.052520088851451874, "p95": 0.1010315865278244 }, { "label": "completed_group_6", "mean": 0.26602888107299805, "median": 0.2639757990837097, "p05": 0.15474572777748108, "p95": 0.3846370279788971 }, { "label": "completed_group_7", "mean": 0.20123262703418732, "median": 0.18140269815921783, "p05": 0.08419936150312424, "p95": 0.3782624900341034 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.7512003183364868, "group": 8, "latest_source_mass": 0.33482813835144043, "layer": 29, "max_source_mass": 0.33482813835144043, "mixer_index": 57, "normalized_entropy": 0.7970056117156376, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.07744461297988892, "median": 0.06970229744911194, "p05": 0.014401955530047417, "p95": 0.17144526541233063 }, { "label": "completed_group_1", "mean": 0.046086348593235016, "median": 0.04394831880927086, "p05": 0.028066720813512802, "p95": 0.0718703418970108 }, { "label": "completed_group_2", "mean": 0.05025433003902435, "median": 0.04861236736178398, "p05": 0.029644077643752098, "p95": 0.07761462777853012 }, { "label": "completed_group_3", "mean": 0.03748488053679466, "median": 0.03434944897890091, "p05": 0.020774798467755318, "p95": 0.05938909202814102 }, { "label": "completed_group_4", "mean": 0.04005061462521553, "median": 0.037498839199543, "p05": 0.019541850313544273, "p95": 0.06922487914562225 }, { "label": "completed_group_5", "mean": 0.041623204946517944, "median": 0.039552345871925354, "p05": 0.022081367671489716, "p95": 0.06821337342262268 }, { "label": "completed_group_6", "mean": 0.198423832654953, "median": 0.18318486213684082, "p05": 0.07548617571592331, "p95": 0.3721975088119507 }, { "label": "completed_group_7", "mean": 0.17380405962467194, "median": 0.1514919549226761, "p05": 0.05777408182621002, "p95": 0.36326974630355835 }, { "label": "current_group_partial", "mean": 0.33482813835144043, "median": 0.33602219820022583, "p05": 0.0795319601893425, "p95": 0.5979493856430054 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.0160694122314453, "group": 8, "latest_source_mass": 0.21711516380310059, "layer": 30, "max_source_mass": 0.21711516380310059, "mixer_index": 58, "normalized_entropy": 0.9175527313077866, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.09385271370410919, "median": 0.09211117774248123, "p05": 0.032377734780311584, "p95": 0.16496515274047852 }, { "label": "completed_group_1", "mean": 0.06208505108952522, "median": 0.06140740588307381, "p05": 0.0407925546169281, "p95": 0.0863042101264 }, { "label": "completed_group_2", "mean": 0.06560768187046051, "median": 0.06516212224960327, "p05": 0.041897669434547424, "p95": 0.09085705131292343 }, { "label": "completed_group_3", "mean": 0.07029052078723907, "median": 0.06780634075403214, "p05": 0.04846132919192314, "p95": 0.09917552024126053 }, { "label": "completed_group_4", "mean": 0.06935097277164459, "median": 0.06708167493343353, "p05": 0.04668639227747917, "p95": 0.09815650433301926 }, { "label": "completed_group_5", "mean": 0.056832700967788696, "median": 0.05595892667770386, "p05": 0.038822539150714874, "p95": 0.0771557092666626 }, { "label": "completed_group_6", "mean": 0.19425815343856812, "median": 0.19145764410495758, "p05": 0.12249615788459778, "p95": 0.27496030926704407 }, { "label": "completed_group_7", "mean": 0.17060703039169312, "median": 0.16317546367645264, "p05": 0.09239593893289566, "p95": 0.2726203501224518 }, { "label": "current_group_partial", "mean": 0.21711516380310059, "median": 0.21198657155036926, "p05": 0.11019522696733475, "p95": 0.3391069769859314 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 1.7837426662445068, "group": 8, "latest_source_mass": 0.29026859998703003, "layer": 30, "max_source_mass": 0.29026859998703003, "mixer_index": 59, "normalized_entropy": 0.8118162725118464, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.08836327493190765, "median": 0.07904411852359772, "p05": 0.02631576545536518, "p95": 0.17534467577934265 }, { "label": "completed_group_1", "mean": 0.04953558370471001, "median": 0.04658038914203644, "p05": 0.025181813165545464, "p95": 0.08318150043487549 }, { "label": "completed_group_2", "mean": 0.04813014343380928, "median": 0.045740917325019836, "p05": 0.026714935898780823, "p95": 0.07843323051929474 }, { "label": "completed_group_3", "mean": 0.0373263955116272, "median": 0.03401809185743332, "p05": 0.01990434154868126, "p95": 0.06290259957313538 }, { "label": "completed_group_4", "mean": 0.037015400826931, "median": 0.03405017405748367, "p05": 0.018646355718374252, "p95": 0.06544196605682373 }, { "label": "completed_group_5", "mean": 0.03390160948038101, "median": 0.03198609501123428, "p05": 0.018189728260040283, "p95": 0.05570102855563164 }, { "label": "completed_group_6", "mean": 0.1840541511774063, "median": 0.17327937483787537, "p05": 0.07966600358486176, "p95": 0.32982927560806274 }, { "label": "completed_group_7", "mean": 0.23140482604503632, "median": 0.22406929731369019, "p05": 0.08831030130386353, "p95": 0.40350067615509033 }, { "label": "current_group_partial", "mean": 0.29026859998703003, "median": 0.2745037376880646, "p05": 0.10280418395996094, "p95": 0.5399085879325867 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 1.9973840713500977, "group": 8, "latest_source_mass": 0.19101998209953308, "layer": 31, "max_source_mass": 0.20831164717674255, "mixer_index": 60, "normalized_entropy": 0.9090486661912383, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.10261696577072144, "median": 0.10389091819524765, "p05": 0.01371697522699833, "p95": 0.21163305640220642 }, { "label": "completed_group_1", "mean": 0.06039813905954361, "median": 0.06085003539919853, "p05": 0.026107093319296837, "p95": 0.09348798543214798 }, { "label": "completed_group_2", "mean": 0.06671924144029617, "median": 0.06656201928853989, "p05": 0.034684643149375916, "p95": 0.09819839894771576 }, { "label": "completed_group_3", "mean": 0.08377155661582947, "median": 0.0804532840847969, "p05": 0.05448959022760391, "p95": 0.12039553374052048 }, { "label": "completed_group_4", "mean": 0.07002593576908112, "median": 0.06737451255321503, "p05": 0.044841229915618896, "p95": 0.10655929893255234 }, { "label": "completed_group_5", "mean": 0.05768897384405136, "median": 0.056077782064676285, "p05": 0.039495617151260376, "p95": 0.08242568373680115 }, { "label": "completed_group_6", "mean": 0.20831164717674255, "median": 0.2021285891532898, "p05": 0.11652229726314545, "p95": 0.31706833839416504 }, { "label": "completed_group_7", "mean": 0.1594475358724594, "median": 0.1546279788017273, "p05": 0.06593266129493713, "p95": 0.2735573351383209 }, { "label": "current_group_partial", "mean": 0.19101998209953308, "median": 0.1713225245475769, "p05": 0.06408411264419556, "p95": 0.3850628137588501 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 1.969987154006958, "group": 8, "latest_source_mass": 0.16864418983459473, "layer": 31, "max_source_mass": 0.24503237009048462, "mixer_index": 61, "normalized_entropy": 0.8965797917640489, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.08120635151863098, "median": 0.061164144426584244, "p05": 0.0259944386780262, "p95": 0.23622018098831177 }, { "label": "completed_group_1", "mean": 0.07894632965326309, "median": 0.07658770680427551, "p05": 0.0496862456202507, "p95": 0.11596276611089706 }, { "label": "completed_group_2", "mean": 0.08615730702877045, "median": 0.08467075228691101, "p05": 0.050409119576215744, "p95": 0.12811316549777985 }, { "label": "completed_group_3", "mean": 0.07609779387712479, "median": 0.0722871869802475, "p05": 0.04099677503108978, "p95": 0.11950768530368805 }, { "label": "completed_group_4", "mean": 0.058728985488414764, "median": 0.056080836802721024, "p05": 0.03323862701654434, "p95": 0.09344407916069031 }, { "label": "completed_group_5", "mean": 0.05076971277594566, "median": 0.04946017265319824, "p05": 0.03047419898211956, "p95": 0.07563400268554688 }, { "label": "completed_group_6", "mean": 0.24503237009048462, "median": 0.24110376834869385, "p05": 0.10613486170768738, "p95": 0.3991503119468689 }, { "label": "completed_group_7", "mean": 0.1544169783592224, "median": 0.1400495171546936, "p05": 0.06199025735259056, "p95": 0.2995939254760742 }, { "label": "current_group_partial", "mean": 0.16864418983459473, "median": 0.15222924947738647, "p05": 0.0543956533074379, "p95": 0.3329208791255951 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.071648359298706, "group": 8, "latest_source_mass": 0.11520501971244812, "layer": 32, "max_source_mass": 0.20955611765384674, "mixer_index": 62, "normalized_entropy": 0.9428478002054053, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.10569609701633453, "median": 0.08689188957214355, "p05": 0.03173588588833809, "p95": 0.24571751058101654 }, { "label": "completed_group_1", "mean": 0.09922641515731812, "median": 0.09795533865690231, "p05": 0.06407676637172699, "p95": 0.14120742678642273 }, { "label": "completed_group_2", "mean": 0.08180510997772217, "median": 0.080217644572258, "p05": 0.054607540369033813, "p95": 0.11442367732524872 }, { "label": "completed_group_3", "mean": 0.10646799206733704, "median": 0.10510502755641937, "p05": 0.07415887713432312, "p95": 0.14216329157352448 }, { "label": "completed_group_4", "mean": 0.0826798751950264, "median": 0.08022409677505493, "p05": 0.05491747707128525, "p95": 0.11860857903957367 }, { "label": "completed_group_5", "mean": 0.06615597754716873, "median": 0.06485182791948318, "p05": 0.04477996751666069, "p95": 0.09238749742507935 }, { "label": "completed_group_6", "mean": 0.20955611765384674, "median": 0.206287682056427, "p05": 0.129981130361557, "p95": 0.2996031939983368 }, { "label": "completed_group_7", "mean": 0.13320739567279816, "median": 0.12465103715658188, "p05": 0.06303662061691284, "p95": 0.23065781593322754 }, { "label": "current_group_partial", "mean": 0.11520501971244812, "median": 0.10386130958795547, "p05": 0.03912060335278511, "p95": 0.22735844552516937 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 1.9781701564788818, "group": 8, "latest_source_mass": 0.16112206876277924, "layer": 32, "max_source_mass": 0.22251944243907928, "mixer_index": 63, "normalized_entropy": 0.9003040366848136, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.09446848928928375, "median": 0.074150450527668, "p05": 0.042592622339725494, "p95": 0.19443637132644653 }, { "label": "completed_group_1", "mean": 0.07056106626987457, "median": 0.06868506222963333, "p05": 0.04818308353424072, "p95": 0.0982733890414238 }, { "label": "completed_group_2", "mean": 0.08033448457717896, "median": 0.08047192543745041, "p05": 0.048296794295310974, "p95": 0.11253741383552551 }, { "label": "completed_group_3", "mean": 0.0658978745341301, "median": 0.06256154924631119, "p05": 0.04223886504769325, "p95": 0.0967884361743927 }, { "label": "completed_group_4", "mean": 0.058234501630067825, "median": 0.0563921295106411, "p05": 0.03529864549636841, "p95": 0.08846980333328247 }, { "label": "completed_group_5", "mean": 0.04316519945859909, "median": 0.042268507182598114, "p05": 0.029443470761179924, "p95": 0.05996086448431015 }, { "label": "completed_group_6", "mean": 0.2036968618631363, "median": 0.19645483791828156, "p05": 0.09398645162582397, "p95": 0.3395216464996338 }, { "label": "completed_group_7", "mean": 0.22251944243907928, "median": 0.2130982130765915, "p05": 0.11867167800664902, "p95": 0.35465875267982483 }, { "label": "current_group_partial", "mean": 0.16112206876277924, "median": 0.14023703336715698, "p05": 0.06008770316839218, "p95": 0.3247341811656952 } ], "sources": 9 }, { "branch": "output", "entropy_mean": 1.6036605834960938, "group": null, "latest_source_mass": 0.31639498472213745, "layer": null, "max_source_mass": 0.3412485122680664, "mixer_index": null, "normalized_entropy": 0.7298573846467136, "offset": null, "source_summaries": [ { "label": "embedding", "mean": 0.09759747982025146, "median": 0.09915056824684143, "p05": 0.025461740791797638, "p95": 0.16398483514785767 }, { "label": "completed_group_1", "mean": 0.04215940088033676, "median": 0.04050712287425995, "p05": 0.025720715522766113, "p95": 0.06398092210292816 }, { "label": "completed_group_2", "mean": 0.041145481169223785, "median": 0.03881259262561798, "p05": 0.023510465398430824, "p95": 0.0662468671798706 }, { "label": "completed_group_3", "mean": 0.029554028064012527, "median": 0.027420252561569214, "p05": 0.01627083122730255, "p95": 0.04924973472952843 }, { "label": "completed_group_4", "mean": 0.0289996936917305, "median": 0.027226522564888, "p05": 0.01560397818684578, "p95": 0.048351090401411057 }, { "label": "completed_group_5", "mean": 0.03647579252719879, "median": 0.0347352996468544, "p05": 0.020744025707244873, "p95": 0.058257561177015305 }, { "label": "completed_group_6", "mean": 0.06642461568117142, "median": 0.057809896767139435, "p05": 0.022837381809949875, "p95": 0.14078694581985474 }, { "label": "completed_group_7", "mean": 0.3412485122680664, "median": 0.3336990475654602, "p05": 0.0819513276219368, "p95": 0.6423859596252441 }, { "label": "completed_group_8", "mean": 0.31639498472213745, "median": 0.30713334679603577, "p05": 0.05841828137636185, "p95": 0.6156357526779175 } ], "sources": 9 } ], "mode": "uniform_group_7_attention_only", "optimizer_state_after": "ed03e6fbd4a12d8b063dcb22e0437754285f54d585374cd52fbd534f05d24637", "optimizer_state_before": "ed03e6fbd4a12d8b063dcb22e0437754285f54d585374cd52fbd534f05d24637", "optimizer_state_unchanged": true, "positions": { "attention_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "5e6ecbcd78543fc42d4098a51169dca446d404ce384015643e84d2c78433c949", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.0933690071105957, 0.24537226557731628, 0.07698935270309448, 0.0523640513420105, 0.09380833059549332, 0.03647654876112938, 0.039120711386203766, 0.06371714919805527, 0.05156813934445381, 0.03728010877966881, 0.03983496502041817, 0.040284499526023865, 0.046871740370988846, 0.04314620420336723, 0.04033593460917473, 0.04471700266003609, 0.08397205173969269, 0.0528092160820961, 0.04443390294909477, 0.07560808956623077, 0.03708760067820549, 0.03599191829562187, 0.033304885029792786, 0.04061548039317131, 0.025487789884209633, 0.021565141156315804, 0.02342291921377182, 0.017244853079319, 0.024166326969861984, 0.016819356009364128, 0.021590016782283783, 0.026201345026493073 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.4649816534983984e-05, "first_to_last_ratio": 0.6133417743310027, "imbalance_abs_log_ratio": 0.4888329546457177, "last_quartile_mean": 2.388524171692552e-05, "mean": 1.9987322446013422e-05, "normalized": [ 0.6780230270576082, 0.7887093448808523, 0.6986534396105439, 0.6484959367934814, 0.8514510444272911, 0.7922837420021595, 0.7438354901152789, 0.6621914219241276, 0.5778172755161638, 0.5171035794693484, 0.4877706074723503, 0.46847840966812265, 0.6435954743790038, 0.6065517389413043, 0.5738945029956215, 0.5387114999019043, 0.6620557302521685, 0.6230978412689845, 0.6302401740034101, 0.6730758779712895, 3.1068416607276204, 2.6507463674741896, 2.1236758661204513, 1.6925432901496313, 2.280059835499728, 2.0230175814879154, 1.5320059361124918, 0.9919891205493465, 1.047464080576941, 0.747458121030106, 0.4943340891809568, 0.4438278924396074 ], "peak_layer": 21, "peak_normalized": 3.1068416607276204, "population_cv": 0.6902279095947693, "rest_mean": 1.491360207081824e-05, "spike_contrast": 3.177328471488953, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.73854124720674e-05, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 1.3551864867622498e-05, 1.5764187992317602e-05, 1.3964211575512309e-05, 1.2961697393620852e-05, 1.701822657196317e-05, 1.583563062013127e-05, 1.486727978772251e-05, 1.3235433470981661e-05, 1.1549020200618543e-05, 1.0335515980841592e-05, 9.74922841123771e-06, 9.363629033032339e-06, 1.2863750271208119e-05, 1.2123345186410006e-05, 1.1470614481368102e-05, 1.0767400453914888e-05, 1.3232721357780974e-05, 1.2454057468858082e-05, 1.2596813576237764e-05, 1.3452984603645746e-05, 6.209744606167078e-05, 5.298132236930542e-05, 4.244659430696629e-05, 3.3829408494057134e-05, 4.557229112833738e-05, 4.04347047151532e-05, 3.0620696634287015e-05, 1.9827206415357068e-05, 2.0936002329108305e-05, 1.4939686479920056e-05, 9.880414836516138e-06, 8.870931196724996e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 5.8386324326420436e-05, "first_to_last_ratio": 0.6122991379428171, "imbalance_abs_log_ratio": 0.49053432839924044, "last_quartile_mean": 9.535588196740719e-05, "mean": 7.990860706286185e-05, "normalized": [ 0.6819595509635074, 0.7890116408345377, 0.7028572831526866, 0.6506277716353583, 0.8411281094430977, 0.7819975172814582, 0.7383488867325747, 0.659379423425708, 0.5812184695899715, 0.5213742912213106, 0.49119793516405585, 0.47025613308535275, 0.6531556441723537, 0.6088555258971414, 0.5748085770015207, 0.5376910312372786, 0.6603412669752641, 0.623721283011235, 0.6330742065277639, 0.673110562991544, 3.110642558067809, 2.659691039770698, 2.1254599956552402, 1.6835970725910054, 2.271730508482628, 2.016146819975117, 1.525997866551267, 0.9872323585963724, 1.0463461969157135, 0.7497494666758743, 0.49950984412082, 0.4497811622537339 ], "peak_layer": 21, "peak_normalized": 3.110642558067809, "population_cv": 0.6894640388963613, "rest_mean": 5.963217928958194e-05, "spike_contrast": 3.176159571811301, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00018940131703857333, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 5.4494437790708616e-05, 6.304882117547095e-05, 5.616434646071866e-05, 5.199075894779526e-05, 6.721337558701634e-05, 6.248833233257756e-05, 5.90004310652148e-05, 5.2690091251861304e-05, 4.644435830414295e-05, 4.166229336988181e-05, 3.925094279111363e-05, 3.757751255761832e-05, 5.2192757721059024e-05, 4.8652796976966783e-05, 4.593215271597728e-05, 4.296614133636467e-05, 5.276695083011873e-05, 4.9840698920888826e-05, 5.0588078011060134e-05, 5.378732748795301e-05, 0.000248567113885656, 0.00021253220620565116, 0.00016984254762064666, 0.00013453389692585915, 0.00018153082055505365, 0.00016110748401843011, 0.00012194036389701068, 7.888836262281984e-05, 8.361206710105762e-05, 5.991143552819267e-05, 3.991513585788198e-05, 3.594138615881093e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.05177757702767849, "first_to_last_ratio": 0.6122991390344411, "imbalance_abs_log_ratio": 0.4905343266164125, "last_quartile_mean": 0.0845625507645309, "mean": 0.07086375274229795, "normalized": [ 0.6819594971670494, 0.7890116576903614, 0.7028572900243015, 0.6506277639771851, 0.8411281061890571, 0.7819975377935527, 0.7383489206963046, 0.6593794194994711, 0.5812184937430185, 0.5213742908892104, 0.49119793583007165, 0.47025608951601966, 0.6531556333371846, 0.6088554666652612, 0.5748086106391191, 0.5376910499368489, 0.6603412357830163, 0.6237212478356517, 0.6330741962444261, 0.6731105875933555, 3.1106423414723765, 2.6596911811608206, 2.1254600623847613, 1.6835971617528775, 2.2717305762602686, 2.0161469291603673, 1.5259978651322903, 0.9872322893517783, 1.046346140564392, 0.7497494085906387, 0.4995098453307215, 0.4497811677882405 ], "peak_layer": 21, "peak_normalized": 3.1106423414723765, "population_cv": 0.6894640460633491, "rest_mean": 0.05288241320738086, "spike_contrast": 3.1761596350033328, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.1679629862308502, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 0.04832620918750763, 0.0559123270213604, 0.0498071052134037, 0.04610592499375343, 0.059605494141578674, 0.055415280163288116, 0.05232217535376549, 0.04672610014677048, 0.04118732362985611, 0.03694653883576393, 0.03480812907218933, 0.03332411125302315, 0.04628505930304527, 0.04314578324556351, 0.04073309525847435, 0.038102805614471436, 0.046794258058071136, 0.044199228286743164, 0.044862013310194016, 0.04769914224743843, 0.22043178975582123, 0.18847569823265076, 0.1506180763244629, 0.11930601298809052, 0.1609833538532257, 0.14287173748016357, 0.10813793540000916, 0.06995898485183716, 0.0741480141878128, 0.05313005670905113, 0.03539714217185974, 0.0318731814622879 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 2.5783426735870307e-06, "first_to_last_ratio": 0.4051793155621585, "imbalance_abs_log_ratio": 0.9034255553879731, "last_quartile_mean": 6.3634607556650735e-06, "mean": 4.6376654765367675e-06, "normalized": [ 0.5938425198486498, 0.5509157055337794, 0.5316403027589413, 0.5418606973454094, 0.5633339100257566, 0.5543598464290257, 0.5504375882365875, 0.5612655820503015, 0.48069315903338145, 0.46391041823801765, 0.4674849725879942, 0.48295519511396434, 0.545572725126613, 0.5506188433276169, 0.562498724611722, 0.5687273398483056, 0.6071823454391313, 0.6355476640639065, 0.6649185390633134, 0.708171681180697, 2.8554116124028597, 2.7124639538444475, 2.3108887646144405, 1.9582909856674189, 2.561022359118694, 2.330448233388041, 1.8019052501789465, 1.1900364040916112, 1.17427137851929, 0.847439670183515, 0.5714808777648555, 0.5004027503627657 ], "peak_layer": 21, "peak_normalized": 2.8554116124028597, "population_cv": 0.7571104801963158, "rest_mean": 3.3669317939778986e-06, "spike_contrast": 3.4154619297376074, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.1499627362354659e-05, "top_five_layers": [ 21, 22, 25, 26, 23 ], "top_five_spike_overlap": 4 }, "values": [ 2.754042952801683e-06, 2.554962748035905e-06, 2.4655698780406965e-06, 2.5129686491709435e-06, 2.612554226288921e-06, 2.5709355213621166e-06, 2.5527453999529826e-06, 2.6029620130429976e-06, 2.229294068456511e-06, 2.1514613308681874e-06, 2.168038918171078e-06, 2.239784635094111e-06, 2.530183792259777e-06, 2.553586000431096e-06, 2.6086809157277457e-06, 2.6375671495770803e-06, 2.815908601405681e-06, 2.9474574603227666e-06, 3.0836697533231927e-06, 3.284263357272721e-06, 1.3242443856142927e-05, 1.2579500435094815e-05, 1.0717129043769091e-05, 9.081898497242946e-06, 1.1877164979523513e-05, 1.0807839316839818e-05, 8.356633770745248e-06, 5.518990747077623e-06, 5.44587783224415e-06, 3.930141701857792e-06, 2.6503371373109985e-06, 2.3207005597214447e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 4.510041662797448e-05, "first_to_last_ratio": 0.7790952827082117, "imbalance_abs_log_ratio": 0.24962192645994336, "last_quartile_mean": 5.788819112240162e-05, "mean": 5.5319728573977045e-05, "normalized": [ 0.773246588042071, 0.8728007742851467, 0.7894489572452644, 0.7388647575986155, 0.93400046667114, 0.8711864297093763, 0.8155353398527732, 0.7270624440771737, 0.6573955972523602, 0.5936586455204987, 0.5589274174588638, 0.5346458941833584, 0.7283267334070248, 0.6736038851052123, 0.6331355784039477, 0.5897507584392466, 0.7475522830930575, 0.7053101570891104, 0.7119641660864052, 0.7492590243662761, 3.12757599297753, 2.5917848400852326, 2.0055953331455663, 1.4979326285185426, 2.046515298578556, 1.7634132643784344, 1.3468913197420114, 0.9125566728558376, 0.8547903891367986, 0.6076938025070345, 0.4388138809478027, 0.4007606792397302 ], "peak_layer": 21, "peak_normalized": 3.12757599297753, "population_cv": 0.6314003058647584, "rest_mean": 4.247447921241271e-05, "spike_contrast": 2.93550568337485, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012468407512642444, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 4.277579137124121e-05, 4.8283101932611316e-05, 4.367210203781724e-05, 4.087379784323275e-05, 5.166865230421536e-05, 4.819379682885483e-05, 4.511519364314154e-05, 4.0220897062681615e-05, 3.6366946005728096e-05, 3.284103513578884e-05, 3.0919713026378304e-05, 2.9576465749414638e-05, 4.0290837205247954e-05, 3.726358409039676e-05, 3.502488834783435e-05, 3.2624851883156225e-05, 4.1354389395564795e-05, 3.9017566450638697e-05, 3.938566442229785e-05, 4.1448805859545246e-05, 0.00017301665502600372, 0.00014337683387566358, 0.00011094898945884779, 8.286522643174976e-05, 0.00011321267083985731, 9.755154314916581e-05, 7.45096622267738e-05, 5.0482387450756505e-05, 4.7286772314691916e-05, 3.3617456210777164e-05, 2.4275064788525924e-05, 2.216997199866455e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 3.4371388210274745e-05, "first_to_last_ratio": 1.0401829820244932, "imbalance_abs_log_ratio": 0.039396641931226675, "last_quartile_mean": 3.304359790945455e-05, "mean": 3.764459196986536e-05, "normalized": [ 0.8816678493771932, 0.9693899446900593, 0.8946905920474307, 0.8423224574320944, 1.0424763733212135, 0.9786842139458749, 0.8939194036944309, 0.8012477867415597, 0.7466870805281207, 0.6772263001976553, 0.6385196996076102, 0.6106569929902169, 0.8246651433527326, 0.754643415303597, 0.7061774647623967, 0.6591357483505617, 0.8611964709305377, 0.8158474066493785, 0.8139089019911299, 0.8385172517429504, 3.1700856931792694, 2.5119090982627332, 1.8435872209780362, 1.20061279727019, 1.7372013044809844, 1.4403252932631443, 1.124260994796992, 0.8454742791043673, 0.6749892741330888, 0.47756573224613846, 0.37232968850825565, 0.35007812612005595 ], "peak_layer": 21, "peak_normalized": 3.1700856931792694, "population_cv": 0.5777243795986192, "rest_mean": 3.0027283914764497e-05, "spike_contrast": 2.623549159192338, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 7.877805546741001e-05, "top_five_layers": [ 21, 22, 23, 25, 26 ], "top_five_spike_overlap": 4 }, "values": [ 3.319002644275315e-05, 3.6492288927547634e-05, 3.3680262276902795e-05, 3.170888521708548e-05, 3.924359771190211e-05, 3.684216790134087e-05, 3.36512312060222e-05, 3.016264599864371e-05, 2.81087304756511e-05, 2.5493907742202282e-05, 2.4036813556449488e-05, 2.2987933334661648e-05, 3.104418283328414e-05, 2.8408243451849557e-05, 2.6583762519294396e-05, 2.481289629940875e-05, 3.2419389754068106e-05, 3.071224273298867e-05, 3.063926851609722e-05, 3.1565639801556244e-05, 0.00011933658242924139, 9.455979306949303e-05, 6.940108869457617e-05, 4.519657886703499e-05, 6.539623427670449e-05, 5.422045796876773e-05, 4.2322346416767687e-05, 3.182753425789997e-05, 2.5409695808775723e-05, 1.7977767129195854e-05, 1.4016199202160351e-05, 1.3178548215364572e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00011894054659933317, "first_to_last_ratio": 0.5872965044539193, "imbalance_abs_log_ratio": 0.5322254683632209, "last_quartile_mean": 0.0002025221428993973, "mean": 0.00016715691776880703, "normalized": [ 0.6661211094099168, 0.7516664198985646, 0.6794847688787287, 0.6368670505133169, 0.8138227690639092, 0.7543555617384852, 0.7348750839572986, 0.655209235185799, 0.574283520415357, 0.5076328918049878, 0.48042990206732117, 0.4627002393108533, 0.6212410383878222, 0.5848271937520697, 0.5567879061482588, 0.5160802271657793, 0.6462716472425695, 0.6188797472493953, 0.6250229433532073, 0.6628553995413206, 3.0917898595017927, 2.682430563691213, 2.2093508810883544, 1.774461887715735, 2.3942315154406133, 2.05601129497247, 1.5755606301881262, 1.0265007204876364, 1.034759406759172, 0.7074185453167906, 0.47005424568596776, 0.4280157940671676 ], "peak_layer": 21, "peak_normalized": 3.0917898595017927, "population_cv": 0.7136969008050548, "rest_mean": 0.00012287726874799572, "spike_contrast": 3.3062829815528016, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00040626702248118817, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 0.00011134675150969997, 0.00012564624194055796, 0.0001135805796366185, 0.00010645673319231719, 0.0001360361056867987, 0.0001260957506019622, 0.00012283945397939533, 0.00010952275624731556, 9.599546319805086e-05, 8.485434955218807e-05, 8.030718163354322e-05, 7.734354585409164e-05, 0.00010384473716840148, 9.77579111349769e-05, 9.307095024269074e-05, 8.626638009445742e-05, 0.00010802877659443766, 0.00010345003101974726, 0.0001044769087457098, 0.00011080086551373824, 0.0005168140633031726, 0.0004483868251554668, 0.00036930828355252743, 0.00029661357984878123, 0.00040021236054599285, 0.0003436765109654516, 0.0002633658587001264, 0.00017158669652417302, 0.0001729671930661425, 0.00011824990360764787, 7.857281889300793e-05, 7.154580089263618e-05 ] } } }, "mlp_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "c0b8018c349b1bd09e5d17858eb312a2323da0e92de9651e2bb08ae4adf115c7", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.13583384454250336, 0.0415157787501812, 0.018140379339456558, 0.039518751204013824, 0.022877270355820656, 0.015532293356955051, 0.02314678393304348, 0.05432315170764923, 0.03743875026702881, 0.04006272554397583, 0.045583780854940414, 0.05995568633079529, 0.038651175796985626, 0.04559725895524025, 0.05632574483752251, 0.07952121645212173, 0.042337141931056976, 0.05044656619429588, 0.04418695345520973, 0.06310147047042847, 0.029010163620114326, 0.04586305841803551, 0.06033914536237717, 0.07656749337911606, 0.045884523540735245, 0.051604680716991425, 0.06502768397331238, 0.06792232394218445, 0.060306549072265625, 0.08025261014699936, 0.09663844108581543, 0.12230555713176727 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.4504446880891919e-05, "first_to_last_ratio": 0.7799742191647813, "imbalance_abs_log_ratio": 0.2484944121975852, "last_quartile_mean": 1.8596059362607775e-05, "mean": 1.7611707221476536e-05, "normalized": [ 0.7676507672326907, 0.8939896961682955, 0.7932198087825575, 0.7343267756286923, 0.9619583586954883, 0.8887492216256591, 0.8203281796960983, 0.7283250023246737, 0.6046493280167995, 0.5631858122251725, 0.5352748293628187, 0.5170960457050374, 0.7096316112759569, 0.6650354714059004, 0.6186095704645309, 0.5873028501612974, 0.745450144778515, 0.6925727248464577, 0.6983918422205476, 0.8123571100302189, 3.187610278372453, 2.479389667371713, 1.9749426351868355, 1.5728169038586937, 2.2967089495305513, 1.7383343678306087, 1.1267493302664318, 0.896921899646201, 0.8482125671940435, 0.5644219544345882, 0.5056326690226588, 0.4701536266378137 ], "peak_layer": 21, "peak_normalized": 3.187610278372453, "population_cv": 0.6407168945212984, "rest_mean": 1.3364371086397484e-05, "spike_contrast": 3.0339865668780526, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.054732235090341e-05, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 1.3519640560843982e-05, 1.5744684787932783e-05, 1.3969955034554005e-05, 1.293274817726342e-05, 1.6941728972597048e-05, 1.565239108458627e-05, 1.4447379726334475e-05, 1.282704670302337e-05, 1.0648906936694402e-05, 9.918663636199199e-06, 9.427103577763774e-06, 9.106944162340369e-06, 1.2497824172896799e-05, 1.1712410014297348e-05, 1.0894770639424678e-05, 1.0343405847379472e-05, 1.3128649698046502e-05, 1.2197388059576042e-05, 1.2299872651055921e-05, 1.4306995581137016e-05, 5.613925895886496e-05, 4.36662849097047e-05, 3.478211147012189e-05, 2.769999082374852e-05, 4.0448965592077e-05, 3.061503593926318e-05, 1.9843979316647165e-05, 1.579632589709945e-05, 1.4938471394998487e-05, 9.940434210875537e-06, 8.905054528440814e-06, 8.280208021460567e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 5.7825178828352364e-05, "first_to_last_ratio": 0.7779250739061138, "imbalance_abs_log_ratio": 0.25112506547474206, "last_quartile_mean": 7.433258133460185e-05, "mean": 7.047180702102196e-05, "normalized": [ 0.7709729438555808, 0.8934323683180992, 0.7975854677004488, 0.735979809657913, 0.9494018075243592, 0.8765675915126135, 0.8153500881635194, 0.7250574287716666, 0.6097717163273327, 0.5677339796856655, 0.5387686261476217, 0.5186143686637487, 0.7208389889619476, 0.6673337822721708, 0.619350913682703, 0.5859757532051905, 0.7426384772052281, 0.6930649435580377, 0.7013130372067459, 0.8131614718072019, 3.1952392586801595, 2.492002483655738, 1.9658096772394542, 1.565757699149687, 2.2868513452605574, 1.730044237677833, 1.1193889906179226, 0.8927016938226912, 0.8503968072224443, 0.5697315386513156, 0.5121108287735412, 0.477051875020861 ], "peak_layer": 21, "peak_normalized": 3.1952392586801595, "population_cv": 0.6399903060661813, "rest_mean": 5.349159780760399e-05, "spike_contrast": 3.0315964267275404, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00016216493677347898, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 5.433185651781969e-05, 6.29617934464477e-05, 5.620728916255757e-05, 5.186582711758092e-05, 6.690606096526608e-05, 6.177330214995891e-05, 5.745919406763278e-05, 5.109610719955526e-05, 4.2971714719897136e-05, 4.000923945568502e-05, 3.7967998650856316e-05, 3.654769170680083e-05, 5.0798826123354957e-05, 4.70282175228931e-05, 4.3646778067341074e-05, 4.1294770198874176e-05, 5.233507545199245e-05, 4.884153895545751e-05, 4.942279701936059e-05, 5.730495831812732e-05, 0.00022517428442370147, 0.0001756159181240946, 0.0001385341602144763, 0.00011034177441615611, 0.00016115854668896645, 0.00012191934365546331, 7.88853649282828e-05, 6.291030149441212e-05, 5.992899968987331e-05, 4.015001104562543e-05, 3.608937549870461e-05, 3.36187076754868e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.05127994529902935, "first_to_last_ratio": 0.7779250603495348, "imbalance_abs_log_ratio": 0.2511250829013292, "last_quartile_mean": 0.06591887562535703, "mean": 0.06249510293127969, "normalized": [ 0.7709729361572227, 0.8934323465048744, 0.7975855149952341, 0.7359798240648382, 0.9494017382316656, 0.8765675001781038, 0.8153500447185824, 0.72505743038531, 0.6097717030934552, 0.5677339621130065, 0.5387686632074455, 0.5186143960317154, 0.7208389983566206, 0.6673337961170833, 0.6193509200702015, 0.5859757836243623, 0.7426384826360108, 0.6930649320090142, 0.7013130729739949, 0.8131614290961915, 3.195239322892832, 2.492002478484546, 1.9658097117551228, 1.565757767080052, 2.286851297213217, 1.7300442590877054, 1.1193889075035017, 0.8927017151266293, 0.850396805194782, 0.5697315298797007, 0.5121108408990834, 0.47705189031789486 ], "peak_layer": 21, "peak_normalized": 3.195239322892832, "population_cv": 0.6399903114138983, "rest_mean": 0.04743688339712443, "spike_contrast": 3.0315964733980745, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.14380948841571808, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 0.048182033002376556, 0.05583514645695686, 0.04984518885612488, 0.045995134860277176, 0.0593329593539238, 0.05478117614984512, 0.050955384969711304, 0.045312538743019104, 0.038107745349407196, 0.03548059239983559, 0.03367040306329727, 0.03241086006164551, 0.04504890739917755, 0.041705094277858734, 0.038706399500370026, 0.0366206169128418, 0.046411268413066864, 0.0433131642639637, 0.043828632682561874, 0.050818607211112976, 0.19968681037425995, 0.1557379513978958, 0.12285348027944565, 0.0978521928191185, 0.14291700720787048, 0.10811929404735565, 0.06995632499456406, 0.055789485573768616, 0.0531456358730793, 0.035605430603027344, 0.03200441971421242, 0.029813406988978386 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 2.5564613963524607e-06, "first_to_last_ratio": 0.5127864811959397, "imbalance_abs_log_ratio": 0.6678957364456058, "last_quartile_mean": 4.9854305643748376e-06, "mean": 4.017316605597898e-06, "normalized": [ 0.6798456113910738, 0.634256282725712, 0.6141889252931932, 0.6263774464349289, 0.6493470564091396, 0.6337703288990618, 0.6286848498363817, 0.6244130859106396, 0.5328976115262944, 0.5232956370742762, 0.5315065927518265, 0.5510685265095274, 0.6186898561236764, 0.61845355782106, 0.6214582534592157, 0.6360205679207201, 0.6971874164737014, 0.7218533935679167, 0.751216018745417, 0.8126154301068895, 2.849470236096618, 2.466135866511509, 2.1822638954717197, 1.8671017276874342, 2.6578486696111074, 2.0511136398527587, 1.3836394756187633, 1.0881204602902037, 0.9693986464129284, 0.6555974436549218, 0.5859051787371239, 0.5362583110742603 ], "peak_layer": 21, "peak_normalized": 2.849470236096618, "population_cv": 0.6845172329121801, "rest_mean": 2.972394642913261e-06, "spike_contrast": 3.2498696722946647, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.659895204094937e-06, "top_five_layers": [ 21, 25, 22, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 2.7311550638842164e-06, 2.5480082967987983e-06, 2.4673913685546722e-06, 2.5163565169350477e-06, 2.6086327125085518e-06, 2.5460560664214427e-06, 2.5256260869355174e-06, 2.5084650587814394e-06, 2.1408184238680406e-06, 2.102244252455421e-06, 2.1352302610466722e-06, 2.21381674236909e-06, 2.48547303272062e-06, 2.4845237476256443e-06, 2.4965945613075746e-06, 2.5550959890097147e-06, 2.800822585413698e-06, 2.8999136247875867e-06, 3.0178725864971057e-06, 3.2645334613334853e-06, 1.1447224096627906e-05, 9.907248568197247e-06, 8.766844985075295e-06, 7.500738774979254e-06, 1.0677419595594984e-05, 8.239972885348834e-06, 5.558517841564026e-06, 4.371324394014664e-06, 3.8943812796787824e-06, 2.6337424969824497e-06, 2.353766603846452e-06, 2.1543194179685088e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 4.472154660106753e-05, "first_to_last_ratio": 0.9720417987461926, "imbalance_abs_log_ratio": 0.02835647262099197, "last_quartile_mean": 4.600784313879558e-05, "mean": 4.972217658405498e-05, "normalized": [ 0.8584499806801998, 0.9698597410369046, 0.8789722728991929, 0.8206596789888326, 1.0350787778085522, 0.9590741035863151, 0.885352358683292, 0.7879817136964605, 0.681766382126936, 0.6367239276507738, 0.6029459097234942, 0.5802806915585426, 0.7901418702640034, 0.7260543475594556, 0.6717457696414673, 0.6320731963115481, 0.8258299629559585, 0.7710861929480181, 0.7758212848251917, 0.8804079385328579, 3.189742032958477, 2.4253290292767957, 1.831101594576619, 1.3811352160670616, 1.9810663889253262, 1.505848772273798, 1.0191911203820818, 0.8378518151933386, 0.6819882949730732, 0.4957650414332143, 0.45050795413999184, 0.43016663832222557 ], "peak_layer": 21, "peak_normalized": 3.189742032958477, "population_cv": 0.5888260381347128, "rest_mean": 3.902569470584341e-05, "spike_contrast": 2.7541643918590832, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00010748317872639745, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 4.268400152795948e-05, 4.8223537305602804e-05, 4.370441456558183e-05, 4.080498547409661e-05, 5.1466369768604636e-05, 4.768725193571299e-05, 4.4021646317560226e-05, 3.918016591342166e-05, 3.3898908441187814e-05, 3.165929956594482e-05, 2.9979782993905246e-05, 2.8852819013991393e-05, 3.9287573599722236e-05, 3.610100247897208e-05, 3.3400661777704954e-05, 3.142805508105084e-05, 4.106206324649975e-05, 3.834008384728804e-05, 3.85755229217466e-05, 4.377579898573458e-05, 0.00015860091662034392, 0.00012059263826813549, 9.104635682888329e-05, 6.867304909974337e-05, 9.85029328148812e-05, 7.487407856388018e-05, 5.0676400860538706e-05, 4.165981590631418e-05, 3.390994243090972e-05, 2.4650516934343614e-05, 2.2400236048270017e-05, 2.138882155122701e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 3.413525109863258e-05, "first_to_last_ratio": 1.2661435324014052, "imbalance_abs_log_ratio": 0.23597569202020474, "last_quartile_mean": 2.6960016953125887e-05, "mean": 3.455649110151171e-05, "normalized": [ 0.9594514782126533, 1.0559172909282037, 0.975745614333522, 0.9164681208815236, 1.1324040871965884, 1.0568988869559277, 0.9522645824571156, 0.853330777927115, 0.764207775136671, 0.7126485068215833, 0.6784321369212497, 0.6535681432024799, 0.8733601164357623, 0.7961738645680428, 0.7366143359639346, 0.6918407028660244, 0.9313375531153064, 0.8733294810385006, 0.8700731804450288, 0.9552676935989919, 3.2233604879047584, 2.3447238190767234, 1.657750512343668, 1.0934524955318834, 1.5873519485805714, 1.2386984933428886, 0.925201525179239, 0.8065268391457353, 0.5218707814257912, 0.41548910124399296, 0.38234915497991745, 0.3638905122386056 ], "peak_layer": 21, "peak_normalized": 3.2233604879047584, "population_cv": 0.5498409341438992, "rest_mean": 2.8276630507240986e-05, "spike_contrast": 2.421354209549141, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.846773831057363e-05, "top_five_layers": [ 21, 22, 23, 25, 26 ], "top_five_spike_overlap": 4 }, "values": [ 3.315527646918781e-05, 3.6488796467892826e-05, 3.371834463905543e-05, 3.166992246406153e-05, 3.9131911762524396e-05, 3.652271698229015e-05, 3.290692256996408e-05, 2.9488117434084415e-05, 2.6408339181216434e-05, 2.4626631784485653e-05, 2.344423410249874e-05, 2.258502172480803e-05, 3.018026109202765e-05, 2.751297506620176e-05, 2.545480674598366e-05, 2.390758709225338e-05, 3.2183757866732776e-05, 3.0179202440194786e-05, 3.0066676117712632e-05, 3.301069955341518e-05, 0.00011138802801724523, 8.102542778942734e-05, 5.728604082833044e-05, 3.7785881431773305e-05, 5.485331348609179e-05, 4.280507346265949e-05, 3.1971718271961436e-05, 2.7870737540069968e-05, 1.8034023014479317e-05, 1.435784542991314e-05, 1.3212645171734039e-05, 1.2574779248097911e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00011736533633666113, "first_to_last_ratio": 0.7483252550806645, "imbalance_abs_log_ratio": 0.28991756252458895, "last_quartile_mean": 0.00015683733181504067, "mean": 0.00014681694233331655, "normalized": [ 0.7533903546620173, 0.8553738803636242, 0.7695201039877265, 0.7197323781193449, 0.9197283450253696, 0.8499510430989242, 0.8134983904428675, 0.713998564251377, 0.5939310180888787, 0.5540924620880455, 0.5316239936482348, 0.5152469497806439, 0.6937265570121143, 0.6435256658940999, 0.6086244271926732, 0.565073083117955, 0.7286649146738354, 0.6901178407895221, 0.6926606630227776, 0.787579642784552, 3.202512487766505, 2.547062021361147, 2.053134041643772, 1.6512239149383219, 2.3386358380362053, 1.7974859991650618, 1.165582334252982, 0.9460699217480808, 0.8058682883282949, 0.5402967965967129, 0.48744638481885566, 0.4646216932994768 ], "peak_layer": 21, "peak_normalized": 3.202512487766505, "population_cv": 0.6634127989495132, "rest_mean": 0.00010988123459456902, "spike_contrast": 3.1513093696133976, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00034626976412255317, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 0.00011061046825489029, 0.0001255833776667714, 0.0001129785887314938, 0.00010566890705376863, 0.00013503170339390635, 0.000124787213280797, 0.00011943534627789631, 0.00010482708603376523, 8.719913603272289e-05, 8.135016105370596e-05, 7.805140921846032e-05, 7.564698171336204e-05, 0.00010185081191593781, 9.44804705795832e-05, 8.935637742979452e-05, 8.29623022582382e-05, 0.00010698035475797951, 0.0001013209912343882, 0.00010169432061957195, 0.00011563003499759361, 0.00047018309123814106, 0.0003739518579095602, 0.0003014348621945828, 0.0002424276462988928, 0.00034335136297158897, 0.00026390139828436077, 0.00017112723435275257, 0.00013889909314457327, 0.00011831511801574379, 7.932472362881526e-05, 7.156538777053356e-05, 6.821433635195717e-05 ] } } }, "post_attention_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "90a633f12d984935028fe5d7212be48e7da7b4a0806751939a551938d16dc540", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.0933690071105957, 0.35791492462158203, 0.3940892517566681, 0.4057151973247528, 0.09380833059549332, 0.11286329478025436, 0.13231655955314636, 0.1621972620487213, 0.05156813934445381, 0.08533531427383423, 0.12003090232610703, 0.16063299775123596, 0.046871740370988846, 0.079384446144104, 0.11459959298372269, 0.15625107288360596, 0.08397205173969269, 0.11654024571180344, 0.15294702351093292, 0.21572710573673248, 0.03708760067820549, 0.06553087383508682, 0.09390194714069366, 0.12852425873279572, 0.025487789884209633, 0.06059965491294861, 0.08860157430171967, 0.12100546807050705, 0.024166326969861984, 0.06877554953098297, 0.12682415544986725, 0.2025238573551178 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.4649731156168855e-05, "first_to_last_ratio": 0.6133378815834789, "imbalance_abs_log_ratio": 0.4888393014494737, "last_quartile_mean": 2.3885254108790832e-05, "mean": 1.9987355329931233e-05, "normalized": [ 0.678009807617627, 0.7887008577141735, 0.6986557939305115, 0.6484857236602893, 0.8514277109004328, 0.7922844406625471, 0.7438434125345945, 0.6621918795820526, 0.5778230593889252, 0.517099952998151, 0.4877787236613259, 0.4684712684199979, 0.6435996029119666, 0.6065509230345245, 0.5738950604193179, 0.5387187587215198, 0.6620661534006986, 0.6231027315814553, 0.6302478737823012, 0.6730732689869516, 3.106807426997161, 2.6507698545150737, 2.1237196958164137, 1.692530130717475, 2.2800586324592786, 2.0229827647217142, 1.5320114242177671, 0.9920052348596025, 1.047446522031893, 0.7474751381901243, 0.49434537981673055, 0.44382079174740263 ], "peak_layer": 21, "peak_normalized": 3.106807426997161, "population_cv": 0.6902262093825834, "rest_mean": 1.4913612748959739e-05, "spike_contrast": 3.177336441868021, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.73855652671773e-05, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 1.3551622942031827e-05, 1.5764044292154722e-05, 1.3964281606604345e-05, 1.2961514585185796e-05, 1.7017788195516914e-05, 1.5835670637898147e-05, 1.4867462596157566e-05, 1.3235464393801522e-05, 1.1549154805834405e-05, 1.0335460501664784e-05, 9.749406672199257e-06, 9.363501703774091e-06, 1.286385395360412e-05, 1.2123348824388813e-05, 1.147064449469326e-05, 1.0767563253466506e-05, 1.3232951459940523e-05, 1.2454175703169312e-05, 1.2596988199220505e-05, 1.3452954590320587e-05, 6.209686398506165e-05, 5.29818789800629e-05, 4.2447540181456134e-05, 3.382920112926513e-05, 4.557234206004068e-05, 4.0434075344819576e-05, 3.062085670535453e-05, 1.982756111829076e-05, 2.0935685824952088e-05, 1.4940051187295467e-05, 9.88065676210681e-06, 8.870803867466748e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 5.838616743858438e-05, "first_to_last_ratio": 0.6122959158510028, "imbalance_abs_log_ratio": 0.49053959069685343, "last_quartile_mean": 9.535612753097666e-05, "mean": 7.990876383701107e-05, "normalized": [ 0.6819505190124153, 0.7890093644347925, 0.7028563139468123, 0.6506197116876543, 0.8411174449439035, 0.7819900646040322, 0.738355541900088, 0.6593840482450527, 0.581225660667183, 0.5213750438692012, 0.49120111440148695, 0.47025457310991964, 0.6531562293298311, 0.6088591116736435, 0.5748132311616531, 0.5377004019377738, 0.6603405177621395, 0.6237192853703563, 0.6330722815905625, 0.6731164811445948, 3.1106397331728215, 2.659683636405231, 2.125481866932072, 1.6835777441348436, 2.2717502717230014, 2.016126656978215, 1.5259935068744965, 0.987243624461313, 1.046329666597485, 0.7497586944958216, 0.4995175597166034, 0.44978009771500005 ], "peak_layer": 21, "peak_normalized": 3.1106397331728215, "population_cv": 0.6894632759983654, "rest_mean": 5.96322605377753e-05, "spike_contrast": 3.1761647126039034, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00018940188165288417, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 5.449382297229022e-05, 6.304876296781003e-05, 5.6164379202527925e-05, 5.1990216888953e-05, 6.721265526721254e-05, 6.248785939533263e-05, 5.900107862544246e-05, 5.2690564189106226e-05, 4.644502405426465e-05, 4.1662435251055285e-05, 3.9251273847185075e-05, 3.757746162591502e-05, 5.2192906878190115e-05, 4.865317896474153e-05, 4.593261473928578e-05, 4.2966974433511496e-05, 5.2766994485864416e-05, 4.9840637075249106e-05, 5.058802344137803e-05, 5.3787905926583335e-05, 0.0002485673758201301, 0.00021253203158266842, 0.0001698446285445243, 0.00013453261635731906, 0.00018153275595977902, 0.0001611061888979748, 0.00012194025475764647, 7.88894176366739e-05, 8.361091022379696e-05, 5.9912290453212336e-05, 3.991583071183413e-05, 3.59413716068957e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.05177743686363101, "first_to_last_ratio": 0.6122959237737405, "imbalance_abs_log_ratio": 0.4905395777574602, "last_quartile_mean": 0.08456276590004563, "mean": 0.07086389057803899, "normalized": [ 0.6819504955334809, 0.7890093870265664, 0.7028562909001672, 0.6506197169703973, 0.8411174807188674, 0.7819900763768914, 0.7383555277078758, 0.6593840247569563, 0.581225669233684, 0.5213750641442202, 0.49120113341490995, 0.4702545965653928, 0.6531562554066145, 0.6088591188026321, 0.5748132752540497, 0.5377004128779654, 0.6603405296345127, 0.6237192986744717, 0.6330722814606942, 0.6731164803840397, 3.110639655490678, 2.659683694791194, 2.1254817924711498, 1.6835778007118374, 2.2717501293316196, 2.0161266058664107, 1.5259935301375303, 0.9872436166618438, 1.046329701254781, 0.7497587270494541, 0.4995175477386066, 0.44978008265050484 ], "peak_layer": 21, "peak_normalized": 3.110639655490678, "population_cv": 0.6894632581522718, "rest_mean": 0.052882485229659965, "spike_contrast": 3.1761646361710856, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.16796347945928575, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 0.048325665295124054, 0.05591227486729622, 0.04980713129043579, 0.04610544443130493, 0.05960485711693764, 0.05541485920548439, 0.05232274532318115, 0.046726517379283905, 0.04118791222572327, 0.03694666549563408, 0.03480842337012291, 0.033324070274829865, 0.04628519341349602, 0.043146125972270966, 0.04073350504040718, 0.03810354322195053, 0.04679429903626442, 0.044199176132678986, 0.044861964881420135, 0.04769965261220932, 0.22043202817440033, 0.18847553431987762, 0.15061990916728973, 0.11930487304925919, 0.1609850525856018, 0.14287057518959045, 0.1081378385424614, 0.06995992362499237, 0.07414699345827103, 0.05313082039356232, 0.03539775684475899, 0.03187316656112671 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 2.578323204716071e-06, "first_to_last_ratio": 0.40516650216522565, "imbalance_abs_log_ratio": 0.9034571799034666, "last_quartile_mean": 6.363613948678903e-06, "mean": 4.637725417921956e-06, "normalized": [ 0.5938101840183757, 0.5508949555920272, 0.5316291170730858, 0.5418667351148689, 0.5633228049781451, 0.5543871474409094, 0.5504259634928217, 0.5612281762465482, 0.4806756209677348, 0.4639005001590106, 0.4675101116271274, 0.48295164952862923, 0.5455571920744754, 0.5506470261496602, 0.5624845906858478, 0.5687206755742695, 0.6071629273990067, 0.6355539617633091, 0.6649122494336146, 0.7081540465796796, 2.855448639643535, 2.7123857522157664, 2.310855367040362, 1.9583853011031942, 2.5610612301840545, 2.330546563645516, 1.8019217709344637, 1.1900469094289818, 1.1742021736366306, 0.8474347965935753, 0.5714919746961272, 0.5004238849786459 ], "peak_layer": 21, "peak_normalized": 2.855448639643535, "population_cv": 0.7571195472719406, "rest_mean": 3.3669652431719307e-06, "spike_contrast": 3.415488290202367, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.1499830361572094e-05, "top_five_layers": [ 21, 22, 25, 26, 23 ], "top_five_spike_overlap": 4 }, "values": [ 2.753928583842935e-06, 2.5548995381541317e-06, 2.4655498691572575e-06, 2.5130291305686114e-06, 2.6125364911422366e-06, 2.571095365055953e-06, 2.5527244815748418e-06, 2.6028221782326e-06, 2.2292415451374836e-06, 2.151443140974152e-06, 2.16818352782866e-06, 2.2397971406462602e-06, 2.5301444566139253e-06, 2.5537497094774153e-06, 2.6086490834131837e-06, 2.6375703328085365e-06, 2.8158549412182765e-06, 2.9475247629306978e-06, 3.0836804398859385e-06, 3.2842240216268692e-06, 1.3242786735645495e-05, 1.2579300346260425e-05, 1.071711267286446e-05, 9.082453289011028e-06, 1.1877498764079064e-05, 1.080843503586948e-05, 8.356818398169708e-06, 5.519110800378257e-06, 5.445627266453812e-06, 3.930169896193547e-06, 2.6504228571866406e-06, 2.3208285711007193e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 4.510041480898508e-05, "first_to_last_ratio": 0.7790928184850469, "imbalance_abs_log_ratio": 0.2496250893940756, "last_quartile_mean": 5.78883718844736e-05, "mean": 5.531981776130124e-05, "normalized": [ 0.7732410668305242, 0.8728012084991762, 0.7894551156669459, 0.7388629087646373, 0.9339921873087328, 0.8711821973766478, 0.8155377077455782, 0.7270625871510766, 0.6573973651854158, 0.5936594640084032, 0.5589294756689257, 0.5346447034070074, 0.7283275978319718, 0.6736029306376706, 0.6331372539239829, 0.5897578964436213, 0.7475518670316339, 0.7053097433683194, 0.711964004688607, 0.7492628143639716, 3.127570424557804, 2.591775663613497, 2.005598149864337, 1.497927714550843, 2.046522126611044, 1.763411473583497, 1.3468934886025272, 0.912559476193609, 0.8547896028967124, 0.6076947956552816, 0.43881646161979826, 0.4007605263482003 ], "peak_layer": 21, "peak_normalized": 3.127570424557804, "population_cv": 0.6313990264054583, "rest_mean": 4.247456820815353e-05, "spike_contrast": 2.93550165683295, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012468416534829884, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 4.277555490261875e-05, 4.8283203796017915e-05, 4.367251312942244e-05, 4.087376146344468e-05, 5.1668277592398226e-05, 4.8193640395766124e-05, 4.5115397369954735e-05, 4.022096982225776e-05, 3.63671024388168e-05, 3.2841133361216635e-05, 3.091987673542462e-05, 2.9576447559520602e-05, 4.029094998259097e-05, 3.7263591366354376e-05, 3.502503750496544e-05, 3.26252993545495e-05, 4.135443305131048e-05, 3.9017606468405575e-05, 3.938571899197996e-05, 4.1449082345934585e-05, 0.00017301662592217326, 0.00014337655738927424, 0.00011094932415289804, 8.286508818855509e-05, 0.0001132132310885936, 9.755160135682672e-05, 7.450990233337507e-05, 5.0482623919378966e-05, 4.728680505650118e-05, 3.3617565350141376e-05, 2.427524668746628e-05, 2.2169999283505604e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 3.436674387558014e-05, "first_to_last_ratio": 1.0400126992484455, "imbalance_abs_log_ratio": 0.0392329238945429, "last_quartile_mean": 3.3044542533389176e-05, "mean": 3.764420125662582e-05, "normalized": [ 0.8815052689840843, 0.9693541981892977, 0.8944302493363351, 0.8422101086302067, 1.0422736163603026, 0.9784999298264467, 0.8941042787457342, 0.8011097882817149, 0.7467471133349635, 0.6773007847237019, 0.6384740440118766, 0.6106197459077125, 0.8245138581835132, 0.7547256131923091, 0.7061368602455695, 0.6592262325050229, 0.8610679856701473, 0.8159047748385688, 0.813901693773824, 0.8384398475418249, 3.1700488209333035, 2.512533764906927, 1.8435497240569074, 1.2008233729127957, 1.7370778524413861, 1.4404299255401702, 1.124391532235005, 0.8455254798371896, 0.6750734961966998, 0.4775614114024039, 0.37240603381903803, 0.3500325934350171 ], "peak_layer": 21, "peak_normalized": 3.1700488209333035, "population_cv": 0.5777711279232812, "rest_mean": 3.0026083550183102e-05, "spike_contrast": 2.6237866400307164, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 7.878203687141649e-05, "top_five_layers": [ 21, 22, 23, 25, 26 ], "top_five_spike_overlap": 4 }, "values": [ 3.318356175441295e-05, 3.649056452559307e-05, 3.367011231603101e-05, 3.1704326829640195e-05, 3.923555777873844e-05, 3.6834848287981004e-05, 3.365784141351469e-05, 3.0157138098729774e-05, 2.8110698622185737e-05, 2.5496447051409632e-05, 2.403484540991485e-05, 2.298629260621965e-05, 3.103816561633721e-05, 2.8411042876541615e-05, 2.658195808180608e-05, 2.4816044970066287e-05, 3.241421654820442e-05, 3.071408355026506e-05, 3.063867916353047e-05, 3.1562398362439126e-05, 0.00011933395580854267, 9.458232671022415e-05, 6.939895683899522e-05, 4.5204036723589525e-05, 6.539090827573091e-05, 5.4223834013100713e-05, 4.232682113070041e-05, 3.182913133059628e-05, 2.541260255384259e-05, 1.7977417883230373e-05, 1.401892768626567e-05, 1.3176697393646464e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00011894031558767892, "first_to_last_ratio": 0.5873010871025878, "imbalance_abs_log_ratio": 0.5322176654378593, "last_quartile_mean": 0.00020252016929589445, "mean": 0.00016715415631551878, "normalized": [ 0.6662205202328879, 0.751711484048231, 0.679585358100091, 0.6366870914110613, 0.8138177576870748, 0.7542107122552005, 0.7350044029140683, 0.6552476565267095, 0.5742567486277126, 0.5075560059127602, 0.48046090902749855, 0.46277800760031923, 0.6212227033603731, 0.5848944434403132, 0.5567137040388535, 0.5160522326659647, 0.6463473553788188, 0.6188950642155443, 0.6248262042506124, 0.6628810192568758, 3.09211464385624, 2.681729845944161, 2.209633925460239, 1.7745343827337179, 2.3942566176615556, 2.0556169413696317, 1.5759411545932325, 1.026517765745547, 1.0346843081870685, 0.707429579226713, 0.4701038419885209, 0.4280676122824023 ], "peak_layer": 21, "peak_normalized": 3.09211464385624, "population_cv": 0.7136838874205806, "rest_mean": 0.0001228752096519909, "spike_contrast": 3.3062850468307365, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00040626046829856933, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 0.00011136152897961438, 0.0001256516989087686, 0.00011359551717760041, 0.00010642489360179752, 0.00013603302068077028, 0.00012606945529114455, 0.0001228590408572927, 0.00010952736920444295, 9.598940232535824e-05, 8.484009595122188e-05, 8.031103789107874e-05, 7.73552674218081e-05, 0.00010383995686424896, 9.776753722690046e-05, 9.305700950790197e-05, 8.626027556601912e-05, 0.00010803964687511325, 0.00010345088230678812, 0.00010444229701533914, 0.00011080331751145422, 0.0005168598145246506, 0.00044826228986494243, 0.00036934949457645416, 0.0002966207975987345, 0.00040020994492806494, 0.00034360491554252803, 0.0002634251140989363, 0.00017158671107608825, 0.00017295178258791566, 0.00011824979446828365, 7.857981108827516e-05, 7.155328057706356e-05 ] } } }, "post_mlp_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "89f2226456bfe96e245d2aad0f699d32c676f9f53099789cd870253d002402f5", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.16915453970432281, 0.374446302652359, 0.39966297149658203, 0.4194087088108063, 0.09892481565475464, 0.11664740741252899, 0.13951337337493896, 0.18593810498714447, 0.06607726216316223, 0.10425819456577301, 0.14551182091236115, 0.19865906238555908, 0.06484340131282806, 0.10430748760700226, 0.14720691740512848, 0.20439592003822327, 0.10052042454481125, 0.14085541665554047, 0.1745898574590683, 0.2420409768819809, 0.049456026405096054, 0.08509506285190582, 0.11783231794834137, 0.15649692714214325, 0.05533855780959129, 0.08507220447063446, 0.11942370235919952, 0.1559232622385025, 0.06628894805908203, 0.12384890764951706, 0.19928614795207977, 0.30097496509552 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.4504459613817744e-05, "first_to_last_ratio": 0.7799755619112818, "imbalance_abs_log_ratio": 0.24849269067229404, "last_quartile_mean": 1.859604367382417e-05, "mean": 1.7611682892493263e-05, "normalized": [ 0.7676555458663804, 0.8940271835080936, 0.7931975625624698, 0.7343277383946722, 0.9619634057479414, 0.8887499329385986, 0.8203246135254102, 0.7283167129612771, 0.6046406612401677, 0.5631849893283694, 0.5352822821999156, 0.5170925254199586, 0.7096415255584291, 0.6650365966598462, 0.6186292225452237, 0.5872929716662602, 0.7454424987706566, 0.6925790522962479, 0.6983910511742576, 0.812362208629548, 3.1875638664835724, 2.4793674782178017, 1.9749780008586828, 1.5728124664471095, 2.2966817569962577, 1.738357632374365, 1.1267553279448252, 0.896919007340079, 0.8482074902965033, 0.564436109296538, 0.5056283066370483, 0.4701542761134935 ], "peak_layer": 21, "peak_normalized": 3.1875638664835724, "population_cv": 0.6407107241197729, "rest_mean": 1.3364394935369663e-05, "spike_contrast": 3.0339598655267634, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.05470378609607e-05, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 1.351970604446251e-05, 1.5745323253213428e-05, 1.3969543942948803e-05, 1.2932747267768718e-05, 1.6941794456215575e-05, 1.5652381989639252e-05, 1.4447296962316614e-05, 1.2826882993977051e-05, 1.0648739589669276e-05, 9.918635441863444e-06, 9.427221812075004e-06, 9.106869583774824e-06, 1.2497981515480205e-05, 1.1712413652276155e-05, 1.0895101695496123e-05, 1.0343217581976205e-05, 1.3128496902936604e-05, 1.2197482647025026e-05, 1.229984172823606e-05, 1.4307065612229053e-05, 5.613836401607841e-05, 4.366583380033262e-05, 3.478268627077341e-05, 2.7699874408426695e-05, 4.044843080919236e-05, 3.0615403375122696e-05, 1.9844057533191517e-05, 1.579625313752331e-05, 1.4938361346139573e-05, 9.940669770003296e-06, 8.90496539796004e-06, 8.280208021460567e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 5.7825492149277125e-05, "first_to_last_ratio": 0.777927471030324, "imbalance_abs_log_ratio": 0.251121984046212, "last_quartile_mean": 7.43327550480899e-05, "mean": 7.047180031349853e-05, "normalized": [ 0.7709770438454313, 0.8934622915563258, 0.7975760449487334, 0.7359825124911049, 0.9493955998600502, 0.8765807872336043, 0.815353934261271, 0.7250554844785851, 0.6097724970901502, 0.5677366665051775, 0.5387701745001272, 0.5186147277647047, 0.7208481432521969, 0.667335239615169, 0.6193593355886976, 0.5859722986019066, 0.7426317852524662, 0.6930619121330119, 0.7013151172621219, 0.8131688280732419, 3.195166051387935, 2.4919855819476275, 1.9658340239964343, 1.5657460780926413, 2.28683112014225, 1.7300690782264534, 1.1193930205237281, 0.8927066313696794, 0.8503960621924582, 0.5697440340661658, 0.5121059733137718, 0.47705192042677674 ], "peak_layer": 21, "peak_normalized": 3.195166051387935, "population_cv": 0.6399809905473468, "rest_mean": 5.3491847480742125e-05, "spike_contrast": 3.0315562697430574, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00016216354561038315, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 5.433214028016664e-05, 6.29638961981982e-05, 5.620661977445707e-05, 5.186601265450008e-05, 6.690561713185161e-05, 6.177422619657591e-05, 5.74594596400857e-05, 5.1095965318381786e-05, 4.2971765651600435e-05, 4.000942499260418e-05, 3.796810415224172e-05, 3.6547713534673676e-05, 5.0799466407625005e-05, 4.702831574832089e-05, 4.364736741990782e-05, 4.129452281631529e-05, 5.233459887676872e-05, 4.884132067672908e-05, 4.9422938900534064e-05, 5.730547127313912e-05, 0.00022516910394188017, 0.00017561471031513065, 0.000138535862788558, 0.0001103409449569881, 0.00016115710604935884, 0.0001219210826093331, 7.888564141467214e-05, 6.291064346441999e-05, 5.9928941482212394e-05, 4.015088779851794e-05, 3.608902989071794e-05, 3.36187076754868e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.05128022423014045, "first_to_last_ratio": 0.7779274618108067, "imbalance_abs_log_ratio": 0.25112199589759615, "last_quartile_mean": 0.0659190306905657, "mean": 0.06249509734334424, "normalized": [ 0.7709770585267884, 0.8934622310505572, 0.7975760488192215, 0.7359825126818341, 0.949395623752104, 0.8765807522153639, 0.8153539326187993, 0.7250554684986782, 0.6097724729274188, 0.5677366356864701, 0.5387702016139136, 0.5186147404496967, 0.720848183035769, 0.6673352268004303, 0.6193593207537281, 0.5859723190688698, 0.7426317535757165, 0.6930619539033541, 0.7013151027887711, 0.8131688337506356, 3.1951661699088225, 2.4919855338200527, 1.9658339696915497, 1.5657461044351926, 2.2868312345206645, 1.7300690920366113, 1.1193929418076851, 0.8927066233016185, 0.8503960467016562, 0.569744039169687, 0.5121059391153361, 0.4770519329730035 ], "peak_layer": 21, "peak_normalized": 3.1951661699088225, "population_cv": 0.6399810058273468, "rest_mean": 0.04743710498291033, "spike_contrast": 3.03155633425555, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.14380825608968734, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 0.04818228632211685, 0.055837009102106094, 0.049844592809677124, 0.04599529877305031, 0.05933257192373276, 0.05478199943900108, 0.050955623388290405, 0.045312412083148956, 0.03810779005289078, 0.03548075631260872, 0.03367049619555473, 0.032410878688097, 0.045049477368593216, 0.0417051799595356, 0.03870692104101181, 0.03662039712071419, 0.04641084372997284, 0.04331297427415848, 0.043828755617141724, 0.05081906542181969, 0.19968222081661224, 0.15573687851428986, 0.12285498529672623, 0.0978514552116394, 0.142915740609169, 0.10812083631753922, 0.06995657086372375, 0.05578978732228279, 0.05314558371901512, 0.03560620918869972, 0.032004110515117645, 0.029813406988978386 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 2.5564826273694052e-06, "first_to_last_ratio": 0.5127758338438722, "imbalance_abs_log_ratio": 0.6679165003756212, "last_quartile_mean": 4.985575486671223e-06, "mean": 4.017343428586173e-06, "normalized": [ 0.6798403364254104, 0.6342462183291464, 0.6141729954924066, 0.6264172974975625, 0.6493611718042315, 0.6337659841469234, 0.6286834821358898, 0.6244043889895178, 0.5329145419593863, 0.5233280828767041, 0.5314853854083773, 0.5510669412632703, 0.6186616710992112, 0.618433581086912, 0.6214752717670565, 0.6359983231705225, 0.6971915907853503, 0.7218623838154119, 0.7512214736615535, 0.8126041748424272, 2.8493176394387327, 2.4660784236569073, 2.182245702664605, 1.867118805592784, 2.657947968395044, 2.0511902754191143, 1.3836852506117079, 1.0881563228199826, 0.9693724778242953, 0.6556200636181531, 0.5858770428152544, 0.5362547305861476 ], "peak_layer": 21, "peak_normalized": 2.8493176394387327, "population_cv": 0.6845126871888862, "rest_mean": 2.9724311321775465e-06, "spike_contrast": 3.2498212404726496, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.659869829192757e-06, "top_five_layers": [ 21, 25, 22, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 2.7311521080264356e-06, 2.5479848773102276e-06, 2.4673438474565046e-06, 2.5165334136545425e-06, 2.6087068363267463e-06, 2.5460556116740918e-06, 2.5256374556192895e-06, 2.508446868887404e-06, 2.140900733138551e-06, 2.1023886347393272e-06, 2.135159320459934e-06, 2.2138251551950816e-06, 2.4853763989085564e-06, 2.4844600829965202e-06, 2.4966795990621904e-06, 2.5550236841809237e-06, 2.8008580557070673e-06, 2.899969103964395e-06, 3.0179146506270627e-06, 3.2645100418449147e-06, 1.144668749475386e-05, 9.907083949656226e-06, 8.766830433160067e-06, 7.500857464037836e-06, 1.06778898043558e-05, 8.240335773734841e-06, 5.5587388487765566e-06, 4.371497652755352e-06, 3.894302153639728e-06, 2.633850954225636e-06, 2.3536692879133625e-06, 2.1543194179685088e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 4.4721616632159567e-05, "first_to_last_ratio": 0.9720403184736837, "imbalance_abs_log_ratio": 0.028357995470768522, "last_quartile_mean": 4.600798524734273e-05, "mean": 4.972221955767964e-05, "normalized": [ 0.8584539945379274, 0.9698681949016709, 0.8789649282810236, 0.8206648961646681, 1.0350733469212077, 0.9590754696651882, 0.8853522519907361, 0.7879805936679553, 0.6817656465618194, 0.6367268893175894, 0.6029466324359721, 0.5802799339557888, 0.7901452846641357, 0.7260535737173992, 0.6717544079923483, 0.6320720646986142, 0.8258239812563558, 0.7710829657061872, 0.7758214191296182, 0.8804140552285782, 3.189711180380848, 2.425314933894289, 1.8311144988825183, 1.3811363637015792, 1.9810582381279151, 1.5058549337456937, 1.0191974829613886, 0.8378578955031543, 0.6819929735044656, 0.49576896633680767, 0.45050573562619894, 0.43016626654035783 ], "peak_layer": 21, "peak_normalized": 3.189711180380848, "population_cv": 0.5888213758676067, "rest_mean": 3.902580034196877e-05, "spike_contrast": 2.7541493673591635, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00010748288332251832, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 4.268423799658194e-05, 4.8223999328911304e-05, 4.370408714748919e-05, 4.0805280150379986e-05, 5.1466144213918597e-05, 4.7687361075077206e-05, 4.402167905936949e-05, 3.918014408554882e-05, 3.38989011652302e-05, 3.165947418892756e-05, 2.9979844839544967e-05, 2.885280628106557e-05, 3.9287777326535434e-05, 3.610099520301446e-05, 3.340112016303465e-05, 3.1428025977220386e-05, 4.1061801312025636e-05, 3.8339956518029794e-05, 3.8575562939513475e-05, 4.377614095574245e-05, 0.000158599519636482, 0.00012059204163961112, 9.10470771486871e-05, 6.867316551506519e-05, 9.850261267274618e-05, 7.48744496377185e-05, 5.067676102044061e-05, 4.166015423834324e-05, 3.3910204365383834e-05, 2.4650733394082636e-05, 2.240014509879984e-05, 2.138882155122701e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 3.4136854537791805e-05, "first_to_last_ratio": 1.2660550532985695, "imbalance_abs_log_ratio": 0.23590580879432238, "last_quartile_mean": 2.6963167556459666e-05, "mean": 3.455893323689452e-05, "normalized": [ 0.9596035843928109, 1.0558766755014144, 0.975641923751664, 0.9163359858520521, 1.13232448600766, 1.0567710395644825, 0.9523678254192874, 0.853372061037709, 0.7640776623958008, 0.7126440441440735, 0.6782505035308789, 0.6535212740257397, 0.8733773513561505, 0.7961309714310681, 0.7364056951080532, 0.6917883921376066, 0.9312759501362757, 0.8734651982029761, 0.8700589617186624, 0.9552788246466914, 3.2243051908280385, 2.3450453114341863, 1.657264819834856, 1.0931496346900684, 1.5876335877819905, 1.2387222288608297, 0.9253453142387785, 0.8064774246211247, 0.5218590096169714, 0.4155249017774698, 0.3822393683377479, 0.36386479761688056 ], "peak_layer": 21, "peak_normalized": 3.2243051908280385, "population_cv": 0.54998717845396, "rest_mean": 2.82776569899094e-05, "spike_contrast": 2.4216230147727513, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.847782497061416e-05, "top_five_layers": [ 21, 22, 23, 25, 26 ], "top_five_spike_overlap": 4 }, "values": [ 3.3162876206915826e-05, 3.6489971535047516e-05, 3.371714410604909e-05, 3.166759415762499e-05, 3.9131926314439625e-05, 3.652087980299257e-05, 3.291281609563157e-05, 2.949162808363326e-05, 2.6405708922538906e-05, 2.4628217943245545e-05, 2.3439613869413733e-05, 2.2584998077945784e-05, 3.0182989576132968e-05, 2.7513437089510262e-05, 2.544939525250811e-05, 2.390746885794215e-05, 3.218390338588506e-05, 3.0186025469447486e-05, 3.0068309570197016e-05, 3.301341712358408e-05, 0.00011142854782519862, 8.104226435534656e-05, 5.7273304264526814e-05, 3.777808524318971e-05, 5.486692316480912e-05, 4.280891880625859e-05, 3.197894693585113e-05, 2.7870999474544078e-05, 1.8034890672424808e-05, 1.436009733879473e-05, 1.3209784810896963e-05, 1.2574779248097911e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00011736215583368903, "first_to_last_ratio": 0.7482920796238891, "imbalance_abs_log_ratio": 0.289961896444839, "last_quartile_mean": 0.00015684003483329434, "mean": 0.00014681607717648149, "normalized": [ 0.7534224477764373, 0.8550608556302807, 0.7698355179300007, 0.7197436566346097, 0.9194721960003663, 0.8501179091465225, 0.8134626455180847, 0.7139422114348084, 0.5938706868901601, 0.5540397759012576, 0.5314024785501694, 0.515293399109774, 0.6935215584623601, 0.6436459201016858, 0.6085983282592226, 0.5651668073462076, 0.7284634424158133, 0.6900455381458291, 0.6925535358669395, 0.7879381797666477, 3.202477638296447, 2.547595410662016, 2.052786941661139, 1.6513380150744505, 2.3386742000593905, 1.7976966087442279, 1.165598817102456, 0.9461231718432641, 0.8057980554109554, 0.540280603152464, 0.4874090158868104, 0.464624431219202 ], "peak_layer": 21, "peak_normalized": 3.202477638296447, "population_cv": 0.6634527382904521, "rest_mean": 0.00010987893458352321, "spike_contrast": 3.1514379757226165, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0003462766471784562, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 0.00011061452823923901, 0.00012553668057080358, 0.00011302423081360757, 0.00010566994023974985, 0.0001349933008896187, 0.00012481097655836493, 0.00011942939454456791, 0.0001048181948135607, 8.718976459931582e-05, 8.134194649755955e-05, 7.801842730259523e-05, 7.565335545223206e-05, 0.00010182011465076357, 9.449756907997653e-05, 8.935201913118362e-05, 8.297557360492647e-05, 0.00010695014498196542, 0.00010130977898370475, 0.00010167799337068573, 0.0001156819926109165, 0.0004701752041000873, 0.00037402796442620456, 0.0003013821260537952, 0.00024244296946562827, 0.0003433549718465656, 0.0002639307640492916, 0.00017112864588852972, 0.00013890609261579812, 0.00011830410949187353, 7.932187872938812e-05, 7.155947969295084e-05, 6.821433635195717e-05 ] } } }, "pre_attention_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "ee695e78937de01b7128d3866b8fc5dd328ebdf2c159be37918d4f02952d5248", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.0626930519938469, 0.09030644595623016, 0.20229491591453552, 0.20300470292568207, 0.19049881398677826, 0.12695962190628052, 0.12132706493139267, 0.11585728824138641, 0.13427142798900604, 0.10485932976007462, 0.10419609397649765, 0.10852954536676407, 0.1286265105009079, 0.09096142649650574, 0.10080574452877045, 0.10566592961549759, 0.1287592649459839, 0.09650235623121262, 0.1065765842795372, 0.11446042358875275, 0.12395420670509338, 0.07221997529268265, 0.07677769660949707, 0.08416415005922318, 0.09094838052988052, 0.059439923614263535, 0.062182098627090454, 0.06896103918552399, 0.0776243805885315, 0.06285123527050018, 0.07231374084949493, 0.08016443997621536 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 9.233986020262819e-06, "first_to_last_ratio": 1.0953805296808754, "imbalance_abs_log_ratio": 0.09110181858916966, "last_quartile_mean": 8.429934410969508e-06, "mean": 1.278566696072403e-05, "normalized": [ 0.6687066457991216, 0.8883771044885506, 0.6363082718884631, 0.5549294587490897, 0.3384995061096416, 0.7372307402676332, 0.9325725422746776, 1.0210866929426317, 0.6338695874649861, 0.522549864195191, 0.4393885415430651, 0.404810552971503, 0.40473490204001017, 0.7171960830958861, 0.6330119968396196, 0.5454117037333844, 0.8056669843361509, 1.1537760793614105, 1.3225787379007963, 1.6114677004552886, 2.6054027156838258, 4.1210626905217, 2.799631911936776, 2.227113629399092, 1.1606492528015881, 1.24725402659777, 0.8107437415808239, 0.5894757215891561, 0.4829009157940162, 0.2950670330423288, 0.32979477631396165, 0.35872988828186114 ], "peak_layer": 22, "peak_normalized": 4.1210626905217, "population_cv": 0.8345770202387011, "rest_mean": 9.038112112749433e-06, "spike_contrast": 3.6536903645180914, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.302246313978685e-05, "top_five_layers": [ 22, 23, 21, 24, 20 ], "top_five_spike_overlap": 4 }, "values": [ 8.549860467610415e-06, 1.1358493793522939e-05, 8.135625648719724e-06, 7.0951432462607045e-06, 4.327941951487446e-06, 9.425986718269996e-06, 1.1923561942239758e-05, 1.3055274393991567e-05, 8.104445441858843e-06, 6.681148533971282e-06, 5.617875558527885e-06, 5.17577291248017e-06, 5.174805664864834e-06, 9.169830263999756e-06, 8.093480573734269e-06, 6.973452400416136e-06, 1.0300989742972888e-05, 1.4751796697964892e-05, 1.6910051272134297e-05, 2.060368933598511e-05, 3.331181142129935e-05, 5.269053508527577e-05, 3.5795161238638684e-05, 2.847513314918615e-05, 1.4839674804534297e-05, 1.5946974599501118e-05, 1.036589947034372e-05, 7.536840257671429e-06, 6.174210284370929e-06, 3.7726288155681686e-06, 4.216646175336791e-06, 4.586600880429614e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 3.6539625625664485e-05, "first_to_last_ratio": 1.0941322575132544, "imbalance_abs_log_ratio": 0.08996159021492689, "last_quartile_mean": 3.339598606544314e-05, "mean": 5.120172039596582e-05, "normalized": [ 0.6164857399402983, 0.8902058839530127, 0.6097162004376416, 0.5594567251110358, 0.3435077684041257, 0.7541811689718532, 0.8945388411247087, 1.0410323410078521, 0.6089648267418464, 0.538259531715159, 0.44198649967313747, 0.41325368533838036, 0.41049125905764033, 0.7540271995346155, 0.6368790522965317, 0.56840421807915, 0.7752044984023767, 1.1878607794821994, 1.282696716075412, 1.6168422261261557, 2.6263769818062688, 4.184181343126687, 2.814298136518519, 2.2132008851275393, 1.1473698727197938, 1.2671825367395342, 0.804870152950098, 0.525524298223546, 0.4764229331148707, 0.30233867138863585, 0.3309425976019582, 0.3632964292094166 ], "peak_layer": 22, "peak_normalized": 4.184181343126687, "population_cv": 0.8444997528097913, "rest_mean": 3.6058475517266845e-05, "spike_contrast": 3.687766629991448, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001329752427409403, "top_five_layers": [ 22, 23, 21, 24, 20 ], "top_five_spike_overlap": 4 }, "values": [ 3.156513048452325e-05, 4.558007276500575e-05, 3.121851841569878e-05, 2.8645146812777966e-05, 1.7588188711670227e-05, 3.861537334159948e-05, 4.5801927626598626e-05, 5.330264684744179e-05, 3.118004678981379e-05, 2.7559814043343067e-05, 2.2630469175055623e-05, 2.115929964929819e-05, 2.1017858671257272e-05, 3.860748984152451e-05, 3.2609303161734715e-05, 2.910327384597622e-05, 3.9691803976893425e-05, 6.082051550038159e-05, 6.56762786093168e-05, 8.278510358650237e-05, 0.00013447501987684518, 0.00021423728321678936, 0.00014409690629690886, 0.00011331969290040433, 5.874731141375378e-05, 6.488192593678832e-05, 4.1210736526409164e-05, 2.690774817892816e-05, 2.4393673811573535e-05, 1.5480260117328726e-05, 1.6944830349530093e-05, 1.860140218923334e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.032403704593889415, "first_to_last_ratio": 1.0941322165064282, "imbalance_abs_log_ratio": 0.08996155273606943, "last_quartile_mean": 0.029615894774906337, "mean": 0.045406197605188936, "normalized": [ 0.6164857271894594, 0.890205872970518, 0.6097161823547601, 0.5594567330844432, 0.3435077687999873, 0.7541811070171827, 0.8945388443909258, 1.041032298361844, 0.6089647855631785, 0.5382595248017545, 0.4419865083443811, 0.4132536726216108, 0.4104912628413459, 0.7540271931250598, 0.6368790717647814, 0.5684042087188725, 0.7752045468438393, 1.1878608186096438, 1.282696717427694, 1.616842300440689, 2.626376974033575, 4.184181318471878, 2.814298124212257, 2.2132008736854196, 1.1473698968263253, 1.2671825909112413, 0.8048701368520813, 0.5255242988397226, 0.47642293674651676, 0.30233867418495897, 0.33094259881871113, 0.36329643114534216 ], "peak_layer": 22, "peak_normalized": 4.184181318471878, "population_cv": 0.8444997517386854, "rest_mean": 0.031977016685737505, "spike_contrast": 3.6877666146643198, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.11792377457022667, "top_five_layers": [ 22, 23, 21, 24, 20 ], "top_five_spike_overlap": 4 }, "values": [ 0.02799227274954319, 0.04042086377739906, 0.02768489345908165, 0.025402802973985672, 0.015597381629049778, 0.03424449637532234, 0.04061760753393173, 0.047269318252801895, 0.027650775387883186, 0.02444031834602356, 0.020068926736712456, 0.018764277920126915, 0.018638847395777702, 0.03423750773072243, 0.028918256983160973, 0.025809073820710182, 0.03519909083843231, 0.05393624305725098, 0.05824238061904907, 0.07341466099023819, 0.11925379186868668, 0.18998776376247406, 0.1277865767478943, 0.10049303621053696, 0.05209770426154137, 0.05753794312477112, 0.036546092480421066, 0.02386206015944481, 0.02163255400955677, 0.013728049583733082, 0.015026845037937164, 0.01649590954184532 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 2.632946646485834e-06, "first_to_last_ratio": 0.4353386333713955, "imbalance_abs_log_ratio": 0.831631083221682, "last_quartile_mean": 6.048042706652268e-06, "mean": 4.295185757285935e-06, "normalized": [ 2.069963826113308, 0.8643122905365216, 0.18814945527579652, 0.2721376320549323, 0.18427347084042306, 0.346685550945373, 0.4020938016279974, 0.5763798326581444, 0.3681381896654166, 0.38288597720443734, 0.3262365158107106, 0.37366712776028466, 0.3228755806188937, 0.5554389081590617, 0.5449346457201981, 0.6638525522126013, 0.4076506109831288, 0.7863704832548073, 0.6353099995439075, 0.9703457871941221, 1.5256098978196801, 2.9228648339386916, 2.7208592332364026, 2.324179364360591, 1.708096934691484, 2.510506759156548, 2.0019211070115195, 1.5514549465771854, 1.2873645652348786, 0.9452587943662876, 0.6157571326238676, 0.6444241928027977 ], "peak_layer": 22, "peak_normalized": 2.9228648339386916, "population_cv": 0.7955607882164473, "rest_mean": 3.3086276803091835e-06, "spike_contrast": 2.9083355102866046, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.622599372960395e-06, "top_five_layers": [ 22, 23, 26, 24, 1 ], "top_five_spike_overlap": 3 }, "values": [ 8.890879144018982e-06, 3.712381840159651e-06, 8.081368605417083e-07, 1.1688816812238656e-06, 7.914887873994303e-07, 1.4890788406773936e-06, 1.7270675698455307e-06, 2.4756584480201127e-06, 1.5812219089639257e-06, 1.6445663959530066e-06, 1.401246436216752e-06, 1.6049697251219186e-06, 1.386810595249699e-06, 2.3857132873672526e-06, 2.340595528949052e-06, 2.851370027201483e-06, 1.7509350982436445e-06, 3.3776072996261064e-06, 2.7287744615023257e-06, 4.167815404798603e-06, 6.552777904289542e-06, 1.2554247405205388e-05, 1.1686595826176926e-05, 9.982782103179488e-06, 7.336593625950627e-06, 1.0783092875499278e-05, 8.59862302604597e-06, 6.663787189609138e-06, 5.529469945031451e-06, 4.060062110511353e-06, 2.6447912659932626e-06, 2.7679216145770624e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 2.422909142296703e-05, "first_to_last_ratio": 1.1144492947562767, "imbalance_abs_log_ratio": 0.10836037685213101, "last_quartile_mean": 2.1740864781349956e-05, "mean": 3.4358177657622946e-05, "normalized": [ 0.4896309478312228, 0.8793760804767221, 0.6290778044177053, 0.5736641919202048, 0.38932927351853297, 0.8025711615541427, 0.9452454132060604, 0.9326363315318001, 0.6447677907431021, 0.5731089366021115, 0.474162258915211, 0.43395410788887484, 0.45851224377260985, 0.8168571269951159, 0.6684563548686675, 0.5779443379391378, 0.7906535019293007, 1.2011580786599243, 1.2820650468346493, 1.574483420006732, 2.4563633377273426, 4.184941967504889, 2.9542742498972223, 2.2045965574717896, 1.1379680328430635, 1.3352090373368544, 0.7791962256231219, 0.4772280221646529, 0.4849870375778677, 0.30186020185197493, 0.2531318887964602, 0.29258903159293254 ], "peak_layer": 22, "peak_normalized": 4.184941967504889, "population_cv": 0.8446013151281049, "rest_mean": 2.4256689997586095e-05, "spike_contrast": 3.6652243579263883, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 8.890621102182194e-05, "top_five_layers": [ 22, 23, 21, 24, 20 ], "top_five_spike_overlap": 4 }, "values": [ 1.6822827092255466e-05, 3.021375960088335e-05, 2.16139669646509e-05, 1.97100562218111e-05, 1.3376644346863031e-05, 2.757488255156204e-05, 3.2476909836987033e-05, 3.2043684768723324e-05, 2.2153046302264556e-05, 1.9690978660946712e-05, 1.629135113034863e-05, 1.4909872334101237e-05, 1.575364512973465e-05, 2.8065722290193662e-05, 2.296694219694473e-05, 1.985711423913017e-05, 2.716541348490864e-05, 4.1269602661486715e-05, 4.4049418647773564e-05, 5.409638106357306e-05, 8.439616794930771e-05, 0.0001437869796063751, 0.00010150347952730954, 7.57459201849997e-05, 3.909850784111768e-05, 4.587534931488335e-05, 2.6771762350108474e-05, 1.6396685168729164e-05, 1.6663270798744634e-05, 1.0371366442996077e-05, 8.697150406078435e-06, 1.0052825928141829e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 1.584801827902993e-05, "first_to_last_ratio": 1.1227107754017347, "imbalance_abs_log_ratio": 0.11574609619349473, "last_quartile_mean": 1.4115851229234977e-05, "mean": 2.2575895755494457e-05, "normalized": [ 0.40168613602094316, 0.8835794718428998, 0.6416439011354959, 0.5818708566210652, 0.44468086759479, 0.8471918614295928, 1.0017837115890245, 0.8134708682770917, 0.6784400931335728, 0.6323903057621078, 0.5201094515566536, 0.46598226166813483, 0.5158448460152834, 0.8793407299751069, 0.7072879995520035, 0.5920087720858835, 0.8048362687674375, 1.2237031355410597, 1.2867908394551848, 1.537448591644762, 2.257083635017179, 4.107390502361403, 3.056248266057319, 2.1170900962900827, 1.0981014822908306, 1.4236880694224456, 0.7702293843657655, 0.43865575966193415, 0.49488096885351435, 0.31235165790028313, 0.21851097852105061, 0.2456782295900993 ], "peak_layer": 22, "peak_normalized": 4.107390502361403, "population_cv": 0.827885168492766, "rest_mean": 1.619116990157097e-05, "spike_contrast": 3.523736438658803, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.7053415366681294e-05, "top_five_layers": [ 22, 23, 21, 24, 20 ], "top_five_spike_overlap": 4 }, "values": [ 9.06842433323618e-06, 1.9947598048020154e-05, 1.4485685824183747e-05, 1.3136255802237429e-05, 1.0039068911282811e-05, 1.9126115148537792e-05, 2.261616464238614e-05, 1.8364833522355184e-05, 1.531639281893149e-05, 1.4276777619670611e-05, 1.1741936759790406e-05, 1.0519966963329352e-05, 1.1645659469650127e-05, 1.9851904653478414e-05, 1.596766014699824e-05, 1.3365128324949183e-05, 1.816989970393479e-05, 2.7626194423646666e-05, 2.9050455850665458e-05, 3.4709279134403914e-05, 5.095568485558033e-05, 9.272801980841905e-05, 6.899754225742072e-05, 4.779520531883463e-05, 2.4790624593151733e-05, 3.214103344362229e-05, 1.7388618289260194e-05, 9.903046702675056e-06, 1.117238116421504e-05, 7.051618467812659e-06, 4.933081072522327e-06, 5.546406100620516e-06 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 7.089648215696798e-05, "first_to_last_ratio": 1.0986265449899153, "imbalance_abs_log_ratio": 0.09406080419583773, "last_quartile_mean": 6.453192168009991e-05, "mean": 0.00010224555938975755, "normalized": [ 0.5219305933305282, 0.8722648082054248, 0.6213125713449594, 0.5538130183023527, 0.3421838989059673, 0.7482277196533812, 0.8735639343777233, 1.013857392352334, 0.6163246413057988, 0.5040433067749736, 0.44517871577910295, 0.3976790948772391, 0.40426623955652163, 0.7466576101077198, 0.623648237385011, 0.5509464155635984, 0.7549348427905129, 1.2215749720733944, 1.3050211992547247, 1.6072231666024437, 2.6249066335483247, 4.327591094224256, 2.9416904667055324, 2.3319878392630886, 1.1573676393000263, 1.2807898204651926, 0.7860057764550519, 0.4893513183892561, 0.47719161554479855, 0.27325156041558446, 0.2615166033720092, 0.32369725377316777 ], "peak_layer": 22, "peak_normalized": 4.327591094224256, "population_cv": 0.883329228180447, "rest_mean": 7.049814781499595e-05, "spike_contrast": 3.882110245104258, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00027368158189347016, "top_five_layers": [ 22, 23, 21, 24, 20 ], "top_five_spike_overlap": 4 }, "values": [ 5.336508547770791e-05, 8.918520325096324e-05, 6.352645141305402e-05, 5.6624921853654087e-05, 3.498678415780887e-05, 7.650296174688265e-05, 8.931803313316777e-05, 0.00010366241622250527, 6.301645771600306e-05, 5.153618985787034e-05, 4.551754682324827e-05, 4.066092151333578e-05, 4.133442780585028e-05, 7.63424250180833e-05, 6.376526289386675e-05, 5.6331824453081936e-05, 7.718873530393466e-05, 0.00012490061635617167, 0.00013343262253329158, 0.00016433143173344433, 0.0002683850470930338, 0.000442476972239092, 0.0003007747873198241, 0.0002384354011155665, 0.00011833570169983432, 0.00013095507165417075, 8.03656002972275e-05, 5.003399928682484e-05, 4.879072366748005e-05, 2.7938758648815565e-05, 2.6738911401480436e-05, 3.309660678496584e-05 ] } } }, "pre_mlp_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "e53fa425d89dfeb0e9395eca9bcd73dc0a55314abf45c5b10b43e2cbfa724345", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.059514161199331284, 0.08157797157764435, 0.14292451739311218, 0.10997853428125381, 0.0991024449467659, 0.1065012738108635, 0.12155740708112717, 0.1095810979604721, 0.06964563578367233, 0.08311934769153595, 0.09363401681184769, 0.0941167026758194, 0.08564268052577972, 0.09493856877088547, 0.09063401818275452, 0.09473779052495956, 0.10505300015211105, 0.09355910867452621, 0.09618847072124481, 0.11416846513748169, 0.047205954790115356, 0.053352247923612595, 0.06512638926506042, 0.07829347252845764, 0.05368548259139061, 0.05310077592730522, 0.061235878616571426, 0.07320034503936768, 0.05304349213838577, 0.0639304369688034, 0.07404901832342148, 0.08157160878181458 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 4.4347703465064114e-06, "first_to_last_ratio": 0.14121764462943923, "imbalance_abs_log_ratio": 1.9574529997652244, "last_quartile_mean": 3.140379772048618e-05, "mean": 1.6738064829269206e-05, "normalized": [ 0.5789716628283216, 0.21273058722622376, 0.14000936948298368, 0.25940508172457016, 0.19219680524073238, 0.15552636052313817, 0.21068109411665237, 0.3700886313600908, 0.4149145706375323, 0.3070645549525893, 0.27781600218941066, 0.2771850963415722, 0.35488127919461615, 0.37391424859116407, 0.42463416670082393, 0.4227549518791402, 0.3531431501565783, 0.448730928468803, 0.41682470316031156, 0.5316243823096949, 2.5744696971133862, 2.950943186939609, 2.5612797385286217, 2.1806859895834685, 2.8273736665554905, 2.9010984190827513, 2.473305892390666, 1.6369118744706832, 1.857815642920562, 1.4707617788906173, 0.9570197819340649, 0.8852367045051297 ], "peak_layer": 22, "peak_normalized": 2.950943186939609, "population_cv": 0.9635596336971493, "rest_mean": 1.1719898591561812e-05, "spike_contrast": 3.740319267305832, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.383616251288913e-05, "top_five_layers": [ 22, 26, 25, 21, 23 ], "top_five_spike_overlap": 4 }, "values": [ 9.690865226730239e-06, 3.560698360161041e-06, 2.3434859031112865e-06, 4.3419390749477316e-06, 3.2170025860978058e-06, 2.6032103050965816e-06, 3.526393811625894e-06, 6.1945675042807125e-06, 6.944866981939413e-06, 5.1396664275671355e-06, 4.650102255254751e-06, 4.639542112272466e-06, 5.94002585785347e-06, 6.258600933506386e-06, 7.107554210961098e-06, 7.076099791447632e-06, 5.910932941333158e-06, 7.5108873716089875e-06, 6.976838903938187e-06, 8.898363375919871e-06, 4.3091640691272914e-05, 4.9393078370485455e-05, 4.287086630938575e-05, 3.650046346592717e-05, 4.7324763727374375e-05, 4.85587734146975e-05, 4.13983543694485e-05, 2.739873707469087e-05, 3.1096238672034815e-05, 2.4617706003482454e-05, 1.6018659152905457e-05, 1.4817149349255487e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 1.7872484818326484e-05, "first_to_last_ratio": 0.14224147158410994, "imbalance_abs_log_ratio": 1.9502291614875096, "last_quartile_mean": 0.000125648902667308, "mean": 6.697146025658185e-05, "normalized": [ 0.5876392414890794, 0.2153138775346923, 0.14066159442745346, 0.2635738300128564, 0.1914037057681296, 0.15447262966877845, 0.20882360574247028, 0.37304896481375177, 0.41742870393417997, 0.31026393678901465, 0.27807776098164605, 0.27776071453788864, 0.35857548095054353, 0.379650554591902, 0.4288545710480788, 0.4241744896833666, 0.35379096682850697, 0.45112131887086937, 0.4185485924777425, 0.5348783458128957, 2.575535490856581, 2.9330743578627705, 2.549763496238146, 2.1643162853391122, 2.8307394903110303, 2.8937570136290565, 2.4777471013579073, 1.6419238059171237, 1.861050945770071, 1.4635410316424478, 0.9602910630576825, 0.8801970320542237 ], "peak_layer": 22, "peak_normalized": 2.9330743578627705, "population_cv": 0.9605011540185703, "rest_mean": 4.699553772769394e-05, "spike_contrast": 3.7203838995450935, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00017484144191257656, "top_five_layers": [ 22, 26, 25, 21, 23 ], "top_five_spike_overlap": 4 }, "values": [ 3.935505810659379e-05, 1.4419884792005178e-05, 9.420312380825635e-06, 1.765192428138107e-05, 1.2818585673812777e-05, 1.0345257578592282e-05, 1.3985221812617965e-05, 2.4983633920783177e-05, 2.7955809855484404e-05, 2.077882891171612e-05, 1.8623273717821576e-05, 1.8602040654513985e-05, 2.4014323571464047e-05, 2.5425752028240822e-05, 2.8721016860799864e-05, 2.8407584977685474e-05, 2.3693897674093023e-05, 3.0212253477657214e-05, 2.8030810426571406e-05, 3.582158387871459e-05, 0.00017248737276531756, 0.00019643227278720587, 0.0001707613846519962, 0.0001449474220862612, 0.00018957875727210194, 0.00019379913283046335, 0.00016593834152445197, 0.00010996203491231427, 0.00012463729945011437, 9.801548003451899e-05, 6.431209476431832e-05, 5.894808055018075e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.015849498216994107, "first_to_last_ratio": 0.1422414714482866, "imbalance_abs_log_ratio": 1.9502291624423882, "last_quartile_mean": 0.11142670316621661, "mean": 0.05939096098882146, "normalized": [ 0.5876392604390701, 0.2153138730852259, 0.1406615869678941, 0.26357382342621105, 0.19140370153991532, 0.15447263107401785, 0.20882360535180763, 0.37304895009003225, 0.4174287086194827, 0.310263927086953, 0.27807775780716476, 0.2777607149524933, 0.3585754686333922, 0.3796505544674394, 0.4288545454621455, 0.42417451747268164, 0.3537909723731883, 0.4511213410985278, 0.4185485984458362, 0.5348782988871813, 2.5755355941508746, 2.9330743774977734, 2.5497634505251985, 2.164316365384914, 2.8307395050312207, 2.8937569236771052, 2.4777470202512597, 1.6419237027414297, 1.861051028376667, 1.4635410726154934, 0.9602910993780638, 0.8801970230893397 ], "peak_layer": 22, "peak_normalized": 2.9330743774977734, "population_cv": 0.9605011545764902, "rest_mean": 0.04167611265761985, "spike_contrast": 3.720383982332897, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.15505114197731018, "top_five_layers": [ 22, 26, 25, 21, 23 ], "top_five_spike_overlap": 4 }, "values": [ 0.03490046039223671, 0.012787697836756706, 0.008354026824235916, 0.015653902664780617, 0.011367649771273136, 0.009174278005957603, 0.012402234598994255, 0.02215573564171791, 0.024791492149233818, 0.01842687278985977, 0.016515305265784264, 0.016496475785970688, 0.021296141669154167, 0.022547811269760132, 0.025470083579421043, 0.025192132219672203, 0.02101198583841324, 0.026792529970407486, 0.024858003482222557, 0.03176693618297577, 0.1529635339975357, 0.17419810593128204, 0.15143290162086487, 0.12854082882404327, 0.168120339512825, 0.17186300456523895, 0.1471557766199112, 0.09751542657613754, 0.11052960902452469, 0.08692111074924469, 0.05703261122107506, 0.05227574706077576 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 1.03075177548817e-06, "first_to_last_ratio": 0.12249561798547509, "imbalance_abs_log_ratio": 2.0996800211843145, "last_quartile_mean": 8.41460121137061e-06, "mean": 4.3323079097490336e-06, "normalized": [ 0.557456905160705, 0.1765725406487582, 0.1366004311756291, 0.28042626004625276, 0.1713297771130422, 0.1426755793358116, 0.16467333740150822, 0.27364193173234375, 0.33779306947757604, 0.2676440127004653, 0.24534833336611897, 0.25010667634169464, 0.3282406949044425, 0.3633888704382868, 0.4394211440658906, 0.42347937776769484, 0.35585180085852586, 0.4396190847097226, 0.4155306819199837, 0.37529367627435534, 2.5564007646851454, 2.9730270085750803, 2.601884653802998, 2.1852680668343054, 2.8435398708348076, 2.949844519467745, 2.7315613611059173, 1.7289255300384423, 1.8830167186728286, 1.46729875021234, 1.0140954475859496, 0.9200431227456325 ], "peak_layer": 22, "peak_normalized": 2.9730270085750803, "population_cv": 0.9980579308084496, "rest_mean": 3.0229688726885123e-06, "spike_contrast": 3.77203312045178, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.1402738709875849e-05, "top_five_layers": [ 22, 26, 25, 27, 23 ], "top_five_spike_overlap": 3 }, "values": [ 2.415074959571939e-06, 7.649666144970979e-07, 5.917951284573064e-07, 1.2148929044997203e-06, 7.422533485623717e-07, 6.181145408845623e-07, 7.134156021493254e-07, 1.1855011052830378e-06, 1.4634235867561074e-06, 1.1595162732191966e-06, 1.06292452528578e-06, 1.0835391321961652e-06, 1.4220397588360356e-06, 1.5743124777145567e-06, 1.9037076981476275e-06, 1.8346430579185835e-06, 1.5416595715578296e-06, 1.9045652379645617e-06, 1.8002068600253551e-06, 1.6258877622021828e-06, 1.1075115253333934e-05, 1.2880068425147329e-05, 1.1272165465925355e-05, 9.467254130868241e-06, 1.2319090274104383e-05, 1.2779634744219948e-05, 1.1833964890684001e-05, 7.490237749152584e-06, 8.157808224495966e-06, 6.356789981509792e-06, 4.393373728817096e-06, 3.985910097981105e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 1.2083814567631634e-05, "first_to_last_ratio": 0.16392673864905302, "imbalance_abs_log_ratio": 1.8083356665196113, "last_quartile_mean": 7.371472565864678e-05, "mean": 4.2019530923198545e-05, "normalized": [ 0.6502576169592559, 0.23174010542181536, 0.11692442467637398, 0.24849575189217293, 0.21911956757778908, 0.17623407606323216, 0.2291368263428325, 0.42870076251407346, 0.48356433772943064, 0.35780524868944985, 0.32528588312814866, 0.33981703050424317, 0.42768519926361304, 0.43666418962368336, 0.4858839430667141, 0.4891102394645643, 0.40985106877619043, 0.5320279164555372, 0.4848808470888936, 0.5801568268317157, 2.667261295327536, 2.9954918573460385, 2.587728580556483, 2.0618025715645074, 2.788167346980433, 2.6972303677185745, 2.318231513101224, 1.5750400753789486, 1.654061285658636, 1.3317246267806577, 0.8710667813591718, 0.7988518361580601 ], "peak_layer": 22, "peak_normalized": 2.9954918573460385, "population_cv": 0.9213936821115557, "rest_mean": 2.941296875010098e-05, "spike_contrast": 3.7430756341978375, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001100949666579254, "top_five_layers": [ 22, 25, 26, 21, 23 ], "top_five_spike_overlap": 4 }, "values": [ 2.7323520043864846e-05, 9.737610525917262e-06, 4.9131094783660956e-06, 1.0441674930916633e-05, 9.207301445712801e-06, 7.405273208860308e-06, 9.628221960156225e-06, 1.8013804947258905e-05, 2.0319146642577834e-05, 1.5034808711789083e-05, 1.366836022498319e-05, 1.427895222150255e-05, 1.797113145585172e-05, 1.8348424418945797e-05, 2.0416615370777436e-05, 2.0552182832034305e-05, 1.7221749658347107e-05, 2.2355563487508334e-05, 2.037446574831847e-05, 2.437791772536002e-05, 0.00011207706847926602, 0.0001258691627299413, 0.00010873514111153781, 8.663597691338509e-05, 0.00011715748405549675, 0.00011333635484334081, 9.741100075189024e-05, 6.61824451526627e-05, 6.95028793415986e-05, 5.5958444136194885e-05, 3.6601817555492744e-05, 3.3567379432497546e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 8.073964011146018e-06, "first_to_last_ratio": 0.2051489589719925, "imbalance_abs_log_ratio": 1.584018934579191, "last_quartile_mean": 3.9356592651529354e-05, "mean": 2.5124759460481982e-05, "normalized": [ 0.7236049665811485, 0.2546503119997429, 0.10878225479391739, 0.24676845590887342, 0.2589007426978255, 0.20979119662023601, 0.25864242563007167, 0.5096986697501894, 0.5824418564835396, 0.42189656810376147, 0.38885978437130053, 0.4247344359349874, 0.5215211280942277, 0.5193363289596591, 0.574841774503572, 0.5841879220333022, 0.49360467660906343, 0.6496187737557957, 0.5795307577907575, 0.6473053590744601, 2.8755832814904605, 3.1411795620414273, 2.6640818383701457, 1.8288645681782367, 2.585445431537255, 2.386828131357412, 2.0106746073493778, 1.4237385551545472, 1.4397615979799785, 1.1901294280534476, 0.7720982041894214, 0.7228964046018586 ], "peak_layer": 22, "peak_normalized": 3.1411795620414273, "population_cv": 0.8699695099066791, "rest_mean": 1.759184041711032e-05, "spike_contrast": 3.7405138254146277, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.580252229468898e-05, "top_five_layers": [ 22, 21, 23, 25, 26 ], "top_five_spike_overlap": 4 }, "values": [ 1.818040072976146e-05, 6.398027835530229e-06, 2.7331279852660373e-06, 6.199998097144999e-06, 6.504818884423003e-06, 5.27095335201011e-06, 6.498328730231151e-06, 1.2806056474801153e-05, 1.4633711543865502e-05, 1.0600049790809862e-05, 9.770008546183817e-06, 1.0671350537450053e-05, 1.3103092896926682e-05, 1.3048200344201177e-05, 1.4442761312238872e-05, 1.4677581020805519e-05, 1.2401698768371716e-05, 1.632151543162763e-05, 1.4560570889443625e-05, 1.6263391444226727e-05, 7.224833825603127e-05, 7.8921380918473e-05, 6.693441537208855e-05, 4.594978236127645e-05, 6.495869456557557e-05, 5.9968482673866674e-05, 5.051771586295217e-05, 3.577108873287216e-05, 3.617366382968612e-05, 2.9901715606683865e-05, 1.9398781660129316e-05, 1.8162598280468956e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 3.53358891516109e-05, "first_to_last_ratio": 0.13136174558034158, "imbalance_abs_log_ratio": 2.0298003448436126, "last_quartile_mean": 0.0002689967996047926, "mean": 0.00014025417681295949, "normalized": [ 0.5773833985388921, 0.20392069284805836, 0.11489944758239863, 0.23429979041732166, 0.18234364659289365, 0.1500430141706743, 0.20408487039615517, 0.34855950453617884, 0.405091259376982, 0.3029768968006166, 0.28038399629256255, 0.2698547152069193, 0.33946456898409383, 0.3631230737311736, 0.41128248692697444, 0.4073824563401681, 0.35149317147152864, 0.4451738738589782, 0.4050264650780412, 0.5095461538871441, 2.4667638861070103, 2.8849144390293584, 2.583310206079801, 2.215288963484183, 2.9966434662882637, 2.9542757763220844, 2.660440011489251, 1.8021538731892146, 1.7816388257256965, 1.3985827119287766, 0.9097984878659665, 0.8398558694526385 ], "peak_layer": 25, "peak_normalized": 2.9966434662882637, "population_cv": 0.9855556946178241, "rest_mean": 9.793418818541492e-05, "spike_contrast": 3.7656116034116662, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00036878211540170014, "top_five_layers": [ 25, 26, 22, 27, 23 ], "top_five_spike_overlap": 3 }, "values": [ 8.098043326754123e-05, 2.860072891053278e-05, 1.6115127436933108e-05, 3.2861524232430384e-05, 2.5574458049959503e-05, 2.1044159439043142e-05, 2.8623755497392267e-05, 4.8886926379054785e-05, 5.681574111804366e-05, 4.249377525411546e-05, 3.9325026591541246e-05, 3.7848250940442085e-05, 4.761132368003018e-05, 5.092952778795734e-05, 5.7684086641529575e-05, 5.7137091062031686e-05, 4.9298385420115665e-05, 6.243749521672726e-05, 5.6806653446983546e-05, 7.146597636165097e-05, 0.0003459739382378757, 0.00040462129982188344, 0.00036232004640623927, 0.00031070352997630835, 0.00042029176256619394, 0.0004143495170865208, 0.00037313782377168536, 0.00025275960797443986, 0.00024988228688016534, 0.000196157066966407, 0.00012760303798131645, 0.00011779329361161217 ] } } } }, "selector": { "expected_uniform_count": 4, "identities_unique": true, "identity_order_sha256": "97fd9721a263024f6a746638f32e0ad9a988cd4c111c00e9e178c1022e5c9d5e", "passed": true, "uniform_count": 4, "uniform_indices": [ 48, 50, 52, 54 ], "visit_count": 65, "visits": [ { "branch": "attention", "coefficient": "detached_learned", "group": 1, "index": 0, "kind": "depth", "layer": 1, "offset": 1, "sources": 1, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 1, "index": 1, "kind": "depth", "layer": 1, "offset": 1, "sources": 2, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 1, "index": 2, "kind": "depth", "layer": 2, "offset": 2, "sources": 2, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 1, "index": 3, "kind": "depth", "layer": 2, "offset": 2, "sources": 2, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 1, "index": 4, "kind": "depth", "layer": 3, "offset": 3, "sources": 2, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 1, "index": 5, "kind": "depth", "layer": 3, "offset": 3, "sources": 2, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 1, "index": 6, "kind": "depth", "layer": 4, "offset": 4, "sources": 2, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 1, "index": 7, "kind": "depth", "layer": 4, "offset": 4, "sources": 2, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 2, "index": 8, "kind": "depth", "layer": 5, "offset": 1, "sources": 2, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 2, "index": 9, "kind": "depth", "layer": 5, "offset": 1, "sources": 3, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 2, "index": 10, "kind": "depth", "layer": 6, "offset": 2, "sources": 3, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 2, "index": 11, "kind": "depth", "layer": 6, "offset": 2, "sources": 3, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 2, "index": 12, "kind": "depth", "layer": 7, "offset": 3, "sources": 3, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 2, "index": 13, "kind": "depth", "layer": 7, "offset": 3, "sources": 3, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 2, "index": 14, "kind": "depth", "layer": 8, "offset": 4, "sources": 3, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 2, "index": 15, "kind": "depth", "layer": 8, "offset": 4, "sources": 3, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 3, "index": 16, "kind": "depth", "layer": 9, "offset": 1, "sources": 3, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 3, "index": 17, "kind": "depth", "layer": 9, "offset": 1, "sources": 4, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 3, "index": 18, "kind": "depth", "layer": 10, "offset": 2, "sources": 4, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 3, "index": 19, "kind": "depth", "layer": 10, "offset": 2, "sources": 4, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 3, "index": 20, "kind": "depth", "layer": 11, "offset": 3, "sources": 4, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 3, "index": 21, "kind": "depth", "layer": 11, "offset": 3, "sources": 4, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 3, "index": 22, "kind": "depth", "layer": 12, "offset": 4, "sources": 4, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 3, "index": 23, "kind": "depth", "layer": 12, "offset": 4, "sources": 4, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 4, "index": 24, "kind": "depth", "layer": 13, "offset": 1, "sources": 4, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 4, "index": 25, "kind": "depth", "layer": 13, "offset": 1, "sources": 5, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 4, "index": 26, "kind": "depth", "layer": 14, "offset": 2, "sources": 5, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 4, "index": 27, "kind": "depth", "layer": 14, "offset": 2, "sources": 5, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 4, "index": 28, "kind": "depth", "layer": 15, "offset": 3, "sources": 5, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 4, "index": 29, "kind": "depth", "layer": 15, "offset": 3, "sources": 5, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 4, "index": 30, "kind": "depth", "layer": 16, "offset": 4, "sources": 5, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 4, "index": 31, "kind": "depth", "layer": 16, "offset": 4, "sources": 5, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 5, "index": 32, "kind": "depth", "layer": 17, "offset": 1, "sources": 5, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 5, "index": 33, "kind": "depth", "layer": 17, "offset": 1, "sources": 6, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 5, "index": 34, "kind": "depth", "layer": 18, "offset": 2, "sources": 6, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 5, "index": 35, "kind": "depth", "layer": 18, "offset": 2, "sources": 6, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 5, "index": 36, "kind": "depth", "layer": 19, "offset": 3, "sources": 6, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 5, "index": 37, "kind": "depth", "layer": 19, "offset": 3, "sources": 6, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 5, "index": 38, "kind": "depth", "layer": 20, "offset": 4, "sources": 6, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 5, "index": 39, "kind": "depth", "layer": 20, "offset": 4, "sources": 6, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 6, "index": 40, "kind": "depth", "layer": 21, "offset": 1, "sources": 6, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 6, "index": 41, "kind": "depth", "layer": 21, "offset": 1, "sources": 7, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 6, "index": 42, "kind": "depth", "layer": 22, "offset": 2, "sources": 7, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 6, "index": 43, "kind": "depth", "layer": 22, "offset": 2, "sources": 7, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 6, "index": 44, "kind": "depth", "layer": 23, "offset": 3, "sources": 7, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 6, "index": 45, "kind": "depth", "layer": 23, "offset": 3, "sources": 7, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 6, "index": 46, "kind": "depth", "layer": 24, "offset": 4, "sources": 7, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 6, "index": 47, "kind": "depth", "layer": 24, "offset": 4, "sources": 7, "use_uniform": false }, { "branch": "attention", "coefficient": "uniform", "group": 7, "index": 48, "kind": "depth", "layer": 25, "offset": 1, "sources": 7, "use_uniform": true }, { "branch": "mlp", "coefficient": "detached_learned", "group": 7, "index": 49, "kind": "depth", "layer": 25, "offset": 1, "sources": 8, "use_uniform": false }, { "branch": "attention", "coefficient": "uniform", "group": 7, "index": 50, "kind": "depth", "layer": 26, "offset": 2, "sources": 8, "use_uniform": true }, { "branch": "mlp", "coefficient": "detached_learned", "group": 7, "index": 51, "kind": "depth", "layer": 26, "offset": 2, "sources": 8, "use_uniform": false }, { "branch": "attention", "coefficient": "uniform", "group": 7, "index": 52, "kind": "depth", "layer": 27, "offset": 3, "sources": 8, "use_uniform": true }, { "branch": "mlp", "coefficient": "detached_learned", "group": 7, "index": 53, "kind": "depth", "layer": 27, "offset": 3, "sources": 8, "use_uniform": false }, { "branch": "attention", "coefficient": "uniform", "group": 7, "index": 54, "kind": "depth", "layer": 28, "offset": 4, "sources": 8, "use_uniform": true }, { "branch": "mlp", "coefficient": "detached_learned", "group": 7, "index": 55, "kind": "depth", "layer": 28, "offset": 4, "sources": 8, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 8, "index": 56, "kind": "depth", "layer": 29, "offset": 1, "sources": 8, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 8, "index": 57, "kind": "depth", "layer": 29, "offset": 1, "sources": 9, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 8, "index": 58, "kind": "depth", "layer": 30, "offset": 2, "sources": 9, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 8, "index": 59, "kind": "depth", "layer": 30, "offset": 2, "sources": 9, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 8, "index": 60, "kind": "depth", "layer": 31, "offset": 3, "sources": 9, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 8, "index": 61, "kind": "depth", "layer": 31, "offset": 3, "sources": 9, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 8, "index": 62, "kind": "depth", "layer": 32, "offset": 4, "sources": 9, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 8, "index": 63, "kind": "depth", "layer": 32, "offset": 4, "sources": 9, "use_uniform": false }, { "branch": "output", "coefficient": "detached_learned", "group": null, "index": 64, "kind": "output", "layer": null, "offset": null, "sources": 9, "use_uniform": false } ], "visits_sha256": "67a61506afe78423707af99e34dfe341cf023966da8e40742d204c35f4eb2249" } }, "uniform_group_7_mlp_only": { "bits_per_byte": 1.8740129107275643, "forward": { "activation_sha256": { "attention_branch_output": "5e6ecbcd78543fc42d4098a51169dca446d404ce384015643e84d2c78433c949", "mlp_branch_output": "c0b8018c349b1bd09e5d17858eb312a2323da0e92de9651e2bb08ae4adf115c7", "post_attention_state": "90a633f12d984935028fe5d7212be48e7da7b4a0806751939a551938d16dc540", "post_mlp_state": "89f2226456bfe96e245d2aad0f699d32c676f9f53099789cd870253d002402f5", "pre_attention_input": "ee695e78937de01b7128d3866b8fc5dd328ebdf2c159be37918d4f02952d5248", "pre_mlp_input": "e53fa425d89dfeb0e9395eca9bcd73dc0a55314abf45c5b10b43e2cbfa724345" }, "logits_sha256": "bdacba5675f0579cec629bbf01b36c8660ad576cfdf9fef1bea8259eaeac9ec2", "loss_nats": 1.2989667654037476, "mixer_summary_sha256": "eb3ac0f07de9b755d6e67a510ab68d09d9ad666bb7b5d020ba7d8c5055c399b2" }, "loss_nats": 1.2989667654037476, "loss_scale": 1.0, "mixers": [ { "branch": "attention", "entropy_mean": -0.0, "group": 1, "latest_source_mass": 1.0, "layer": 1, "max_source_mass": 1.0, "mixer_index": 0, "normalized_entropy": 1.0, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 1.0, "median": 1.0, "p05": 1.0, "p95": 1.0 } ], "sources": 1 }, { "branch": "mlp", "entropy_mean": 0.22886337339878082, "group": 1, "latest_source_mass": 0.06350618600845337, "layer": 1, "max_source_mass": 0.9364938139915466, "mixer_index": 1, "normalized_entropy": 0.3301800538435402, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.9364938139915466, "median": 0.9452826976776123, "p05": 0.8725647330284119, "p95": 0.9712384939193726 }, { "label": "current_group_partial", "mean": 0.06350618600845337, "median": 0.0547172985970974, "p05": 0.028761589899659157, "p95": 0.12743529677391052 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.603729784488678, "group": 1, "latest_source_mass": 0.4133155345916748, "layer": 2, "max_source_mass": 0.5866844654083252, "mixer_index": 2, "normalized_entropy": 0.8709979661187783, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.5866844654083252, "median": 0.5734395384788513, "p05": 0.29813364148139954, "p95": 0.8843642473220825 }, { "label": "current_group_partial", "mean": 0.4133155345916748, "median": 0.4265604615211487, "p05": 0.11563576757907867, "p95": 0.7018663287162781 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.3967982828617096, "group": 1, "latest_source_mass": 0.14110636711120605, "layer": 2, "max_source_mass": 0.858893632888794, "mixer_index": 3, "normalized_entropy": 0.5724589149178446, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.858893632888794, "median": 0.8705973029136658, "p05": 0.7599387168884277, "p95": 0.9270997643470764 }, { "label": "current_group_partial", "mean": 0.14110636711120605, "median": 0.12940272688865662, "p05": 0.07290027290582657, "p95": 0.2400612235069275 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6852496862411499, "group": 1, "latest_source_mass": 0.518975019454956, "layer": 3, "max_source_mass": 0.518975019454956, "mixer_index": 4, "normalized_entropy": 0.9886063241108252, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.48102498054504395, "median": 0.48102718591690063, "p05": 0.38786420226097107, "p95": 0.580143928527832 }, { "label": "current_group_partial", "mean": 0.518975019454956, "median": 0.5189728140830994, "p05": 0.41985607147216797, "p95": 0.6121357679367065 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.5890734195709229, "group": 1, "latest_source_mass": 0.3123423159122467, "layer": 3, "max_source_mass": 0.6876576542854309, "mixer_index": 5, "normalized_entropy": 0.8498533011344741, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.6876576542854309, "median": 0.6909762620925903, "p05": 0.48803940415382385, "p95": 0.8900076746940613 }, { "label": "current_group_partial", "mean": 0.3123423159122467, "median": 0.3090237081050873, "p05": 0.10999235510826111, "p95": 0.5119606256484985 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6873323321342468, "group": 1, "latest_source_mass": 0.4831485152244568, "layer": 4, "max_source_mass": 0.5168514251708984, "mixer_index": 6, "normalized_entropy": 0.9916109470127238, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.5168514251708984, "median": 0.5219584703445435, "p05": 0.4251725673675537, "p95": 0.5947866439819336 }, { "label": "current_group_partial", "mean": 0.4831485152244568, "median": 0.47804147005081177, "p05": 0.405213326215744, "p95": 0.5748274326324463 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.487301766872406, "group": 1, "latest_source_mass": 0.2060384750366211, "layer": 4, "max_source_mass": 0.7939615249633789, "mixer_index": 7, "normalized_entropy": 0.7030278424832499, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.7939615249633789, "median": 0.7953404188156128, "p05": 0.6590759754180908, "p95": 0.9329437017440796 }, { "label": "current_group_partial", "mean": 0.2060384750366211, "median": 0.20465964078903198, "p05": 0.06705628335475922, "p95": 0.3409239649772644 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.674697756767273, "group": 2, "latest_source_mass": 0.42952626943588257, "layer": 5, "max_source_mass": 0.5704737305641174, "mixer_index": 8, "normalized_entropy": 0.9733831077870527, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.5704737305641174, "median": 0.5696132779121399, "p05": 0.4695202708244324, "p95": 0.6750621199607849 }, { "label": "completed_group_1", "mean": 0.42952626943588257, "median": 0.4303867816925049, "p05": 0.3249378800392151, "p95": 0.5304797887802124 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.9691112041473389, "group": 2, "latest_source_mass": 0.2681158185005188, "layer": 5, "max_source_mass": 0.5506307482719421, "mixer_index": 9, "normalized_entropy": 0.8821230329784768, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.5506307482719421, "median": 0.543993353843689, "p05": 0.41262108087539673, "p95": 0.7161091566085815 }, { "label": "completed_group_1", "mean": 0.18125343322753906, "median": 0.1794438362121582, "p05": 0.11575750261545181, "p95": 0.25031501054763794 }, { "label": "current_group_partial", "mean": 0.2681158185005188, "median": 0.2664111256599426, "p05": 0.1564716249704361, "p95": 0.3817881643772125 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0745112895965576, "group": 2, "latest_source_mass": 0.412794291973114, "layer": 6, "max_source_mass": 0.412794291973114, "mixer_index": 10, "normalized_entropy": 0.9780623252441762, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.34262996912002563, "median": 0.3412924110889435, "p05": 0.29556700587272644, "p95": 0.39125555753707886 }, { "label": "completed_group_1", "mean": 0.24457570910453796, "median": 0.24382859468460083, "p05": 0.22120657563209534, "p95": 0.2734297513961792 }, { "label": "current_group_partial", "mean": 0.412794291973114, "median": 0.4151499271392822, "p05": 0.37735897302627563, "p95": 0.43749260902404785 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0074045658111572, "group": 2, "latest_source_mass": 0.3775783181190491, "layer": 6, "max_source_mass": 0.4399106502532959, "mixer_index": 11, "normalized_entropy": 0.9169791528842925, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.4399106502532959, "median": 0.44856905937194824, "p05": 0.22063353657722473, "p95": 0.6359403133392334 }, { "label": "completed_group_1", "mean": 0.18251106142997742, "median": 0.18562838435173035, "p05": 0.1196163073182106, "p95": 0.23045453429222107 }, { "label": "current_group_partial", "mean": 0.3775783181190491, "median": 0.3675365447998047, "p05": 0.23817461729049683, "p95": 0.5557451844215393 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.060497522354126, "group": 2, "latest_source_mass": 0.4025024175643921, "layer": 7, "max_source_mass": 0.4025024175643921, "mixer_index": 12, "normalized_entropy": 0.9653064445872968, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.3786153197288513, "median": 0.3907395303249359, "p05": 0.28039106726646423, "p95": 0.4436405599117279 }, { "label": "completed_group_1", "mean": 0.2188822478055954, "median": 0.21226292848587036, "p05": 0.1885872334241867, "p95": 0.2672030031681061 }, { "label": "current_group_partial", "mean": 0.4025024175643921, "median": 0.39815032482147217, "p05": 0.3533596992492676, "p95": 0.460100919008255 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 0.9941019415855408, "group": 2, "latest_source_mass": 0.4801652431488037, "layer": 7, "max_source_mass": 0.4801652431488037, "mixer_index": 13, "normalized_entropy": 0.9048705824970601, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.32296955585479736, "median": 0.2803739905357361, "p05": 0.1784922033548355, "p95": 0.6440245509147644 }, { "label": "completed_group_1", "mean": 0.19686520099639893, "median": 0.2053825855255127, "p05": 0.09813719242811203, "p95": 0.2504696547985077 }, { "label": "current_group_partial", "mean": 0.4801652431488037, "median": 0.5057140588760376, "p05": 0.2529003322124481, "p95": 0.598870575428009 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0503522157669067, "group": 2, "latest_source_mass": 0.4001518487930298, "layer": 8, "max_source_mass": 0.40130966901779175, "mixer_index": 14, "normalized_entropy": 0.9560717885654542, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.40130966901779175, "median": 0.396759569644928, "p05": 0.3479611873626709, "p95": 0.4733521044254303 }, { "label": "completed_group_1", "mean": 0.19853851199150085, "median": 0.19769755005836487, "p05": 0.17136667668819427, "p95": 0.22788912057876587 }, { "label": "current_group_partial", "mean": 0.4001518487930298, "median": 0.40291696786880493, "p05": 0.3506356179714203, "p95": 0.43941178917884827 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0129402875900269, "group": 2, "latest_source_mass": 0.42161279916763306, "layer": 8, "max_source_mass": 0.42161279916763306, "mixer_index": 15, "normalized_entropy": 0.9220179839951123, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.4093267023563385, "median": 0.4046899676322937, "p05": 0.2895161807537079, "p95": 0.5762131810188293 }, { "label": "completed_group_1", "mean": 0.16906049847602844, "median": 0.16888955235481262, "p05": 0.11233551055192947, "p95": 0.22477403283119202 }, { "label": "current_group_partial", "mean": 0.42161279916763306, "median": 0.4227708578109741, "p05": 0.3131049573421478, "p95": 0.519075334072113 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0774019956588745, "group": 3, "latest_source_mass": 0.3560434877872467, "layer": 9, "max_source_mass": 0.38510820269584656, "mixer_index": 16, "normalized_entropy": 0.9806935592947451, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.38510820269584656, "median": 0.3892287313938141, "p05": 0.2934730052947998, "p95": 0.4718902111053467 }, { "label": "completed_group_1", "mean": 0.25884830951690674, "median": 0.25734007358551025, "p05": 0.22217319905757904, "p95": 0.29551801085472107 }, { "label": "completed_group_2", "mean": 0.3560434877872467, "median": 0.35412120819091797, "p05": 0.29444968700408936, "p95": 0.41908568143844604 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.2740377187728882, "group": 3, "latest_source_mass": 0.3633861243724823, "layer": 9, "max_source_mass": 0.3633861243724823, "mixer_index": 17, "normalized_entropy": 0.9190239493895668, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.29550763964653015, "median": 0.30360478162765503, "p05": 0.16904757916927338, "p95": 0.45146965980529785 }, { "label": "completed_group_1", "mean": 0.09541244804859161, "median": 0.09486871212720871, "p05": 0.061360105872154236, "p95": 0.13037073612213135 }, { "label": "completed_group_2", "mean": 0.24569380283355713, "median": 0.24333015084266663, "p05": 0.1698218435049057, "p95": 0.31589236855506897 }, { "label": "current_group_partial", "mean": 0.3633861243724823, "median": 0.36472851037979126, "p05": 0.2795379161834717, "p95": 0.4399876892566681 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.360302448272705, "group": 3, "latest_source_mass": 0.28569501638412476, "layer": 10, "max_source_mass": 0.3109527826309204, "mixer_index": 18, "normalized_entropy": 0.9812507981160736, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.3109527826309204, "median": 0.32172977924346924, "p05": 0.23520725965499878, "p95": 0.37052470445632935 }, { "label": "completed_group_1", "mean": 0.20984819531440735, "median": 0.20958667993545532, "p05": 0.1891367882490158, "p95": 0.23184512555599213 }, { "label": "completed_group_2", "mean": 0.19350400567054749, "median": 0.19153150916099548, "p05": 0.16607457399368286, "p95": 0.22877082228660583 }, { "label": "current_group_partial", "mean": 0.28569501638412476, "median": 0.28060320019721985, "p05": 0.23824527859687805, "p95": 0.3439165949821472 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.2722704410552979, "group": 3, "latest_source_mass": 0.29912811517715454, "layer": 10, "max_source_mass": 0.35036465525627136, "mixer_index": 19, "normalized_entropy": 0.9177491279900463, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.35036465525627136, "median": 0.36210548877716064, "p05": 0.12807878851890564, "p95": 0.5797606706619263 }, { "label": "completed_group_1", "mean": 0.12631990015506744, "median": 0.1264108121395111, "p05": 0.06977735459804535, "p95": 0.1755298376083374 }, { "label": "completed_group_2", "mean": 0.22418735921382904, "median": 0.22563719749450684, "p05": 0.1176326721906662, "p95": 0.32001155614852905 }, { "label": "current_group_partial", "mean": 0.29912811517715454, "median": 0.28589946031570435, "p05": 0.193109393119812, "p95": 0.44119665026664734 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3558036088943481, "group": 3, "latest_source_mass": 0.314197838306427, "layer": 11, "max_source_mass": 0.314197838306427, "mixer_index": 20, "normalized_entropy": 0.978005571485618, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.298976868391037, "median": 0.2961939573287964, "p05": 0.24950163066387177, "p95": 0.35835936665534973 }, { "label": "completed_group_1", "mean": 0.1941717267036438, "median": 0.1918032318353653, "p05": 0.16809208691120148, "p95": 0.22895419597625732 }, { "label": "completed_group_2", "mean": 0.1926535815000534, "median": 0.19306811690330505, "p05": 0.16983944177627563, "p95": 0.2155935913324356 }, { "label": "current_group_partial", "mean": 0.314197838306427, "median": 0.31541913747787476, "p05": 0.2673022747039795, "p95": 0.35941022634506226 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.2696614265441895, "group": 3, "latest_source_mass": 0.2862822115421295, "layer": 11, "max_source_mass": 0.3667730689048767, "mixer_index": 21, "normalized_entropy": 0.9158671218416545, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.3667730689048767, "median": 0.4067659378051758, "p05": 0.10593366622924805, "p95": 0.5672274827957153 }, { "label": "completed_group_1", "mean": 0.15310388803482056, "median": 0.15050218999385834, "p05": 0.08651046454906464, "p95": 0.22459912300109863 }, { "label": "completed_group_2", "mean": 0.19384080171585083, "median": 0.18732509016990662, "p05": 0.1267009675502777, "p95": 0.281107634305954 }, { "label": "current_group_partial", "mean": 0.2862822115421295, "median": 0.26271265745162964, "p05": 0.17024393379688263, "p95": 0.475044310092926 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3606936931610107, "group": 3, "latest_source_mass": 0.3028184771537781, "layer": 12, "max_source_mass": 0.3028184771537781, "mixer_index": 22, "normalized_entropy": 0.9815330216461395, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.298227459192276, "median": 0.2954302132129669, "p05": 0.25287362933158875, "p95": 0.35405808687210083 }, { "label": "completed_group_1", "mean": 0.18611131608486176, "median": 0.1850002110004425, "p05": 0.15901757776737213, "p95": 0.21725836396217346 }, { "label": "completed_group_2", "mean": 0.21284276247024536, "median": 0.21402311325073242, "p05": 0.18819917738437653, "p95": 0.23418807983398438 }, { "label": "current_group_partial", "mean": 0.3028184771537781, "median": 0.3041122555732727, "p05": 0.25661808252334595, "p95": 0.34458184242248535 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.269316554069519, "group": 3, "latest_source_mass": 0.2653314769268036, "layer": 12, "max_source_mass": 0.4081852436065674, "mixer_index": 23, "normalized_entropy": 0.9156183489371815, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.4081852436065674, "median": 0.4254342317581177, "p05": 0.20765496790409088, "p95": 0.587238073348999 }, { "label": "completed_group_1", "mean": 0.14420127868652344, "median": 0.1438048928976059, "p05": 0.1064920499920845, "p95": 0.1871720254421234 }, { "label": "completed_group_2", "mean": 0.18228203058242798, "median": 0.17507661879062653, "p05": 0.12183455377817154, "p95": 0.26505547761917114 }, { "label": "current_group_partial", "mean": 0.2653314769268036, "median": 0.24924281239509583, "p05": 0.15609969198703766, "p95": 0.4061311185359955 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3636136054992676, "group": 4, "latest_source_mass": 0.3136237859725952, "layer": 13, "max_source_mass": 0.3136237859725952, "mixer_index": 24, "normalized_entropy": 0.9836392931712563, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.27427661418914795, "median": 0.27403637766838074, "p05": 0.22928935289382935, "p95": 0.319773405790329 }, { "label": "completed_group_1", "mean": 0.21341991424560547, "median": 0.20697325468063354, "p05": 0.18552586436271667, "p95": 0.26688331365585327 }, { "label": "completed_group_2", "mean": 0.19867971539497375, "median": 0.197545126080513, "p05": 0.1794913411140442, "p95": 0.222100168466568 }, { "label": "completed_group_3", "mean": 0.3136237859725952, "median": 0.31875866651535034, "p05": 0.24156072735786438, "p95": 0.3695686459541321 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.570773959159851, "group": 4, "latest_source_mass": 0.2294243425130844, "layer": 13, "max_source_mass": 0.2658740282058716, "mixer_index": 25, "normalized_entropy": 0.9759767351225285, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.2658740282058716, "median": 0.2747163474559784, "p05": 0.17927880585193634, "p95": 0.3512483835220337 }, { "label": "completed_group_1", "mean": 0.13419361412525177, "median": 0.13374894857406616, "p05": 0.11072897166013718, "p95": 0.1586270034313202 }, { "label": "completed_group_2", "mean": 0.1907704919576645, "median": 0.19074805080890656, "p05": 0.14699944853782654, "p95": 0.23316305875778198 }, { "label": "completed_group_3", "mean": 0.17973749339580536, "median": 0.17616114020347595, "p05": 0.1363743096590042, "p95": 0.23202598094940186 }, { "label": "current_group_partial", "mean": 0.2294243425130844, "median": 0.22995705902576447, "p05": 0.17975632846355438, "p95": 0.2789157032966614 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.5560038089752197, "group": 4, "latest_source_mass": 0.32468360662460327, "layer": 14, "max_source_mass": 0.32468360662460327, "mixer_index": 26, "normalized_entropy": 0.9667995248241249, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.20477107167243958, "median": 0.206403911113739, "p05": 0.16619175672531128, "p95": 0.2431035190820694 }, { "label": "completed_group_1", "mean": 0.15721669793128967, "median": 0.15658551454544067, "p05": 0.13878680765628815, "p95": 0.17778633534908295 }, { "label": "completed_group_2", "mean": 0.13811248540878296, "median": 0.13743364810943604, "p05": 0.12352626770734787, "p95": 0.15553249418735504 }, { "label": "completed_group_3", "mean": 0.1752161681652069, "median": 0.17309701442718506, "p05": 0.15223677456378937, "p95": 0.20474471151828766 }, { "label": "current_group_partial", "mean": 0.32468360662460327, "median": 0.32497870922088623, "p05": 0.2946181893348694, "p95": 0.35203880071640015 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.5659973621368408, "group": 4, "latest_source_mass": 0.26703086495399475, "layer": 14, "max_source_mass": 0.26703086495399475, "mixer_index": 27, "normalized_entropy": 0.9730088685238187, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.20531430840492249, "median": 0.20229148864746094, "p05": 0.10698921978473663, "p95": 0.3393132984638214 }, { "label": "completed_group_1", "mean": 0.15986156463623047, "median": 0.15952220559120178, "p05": 0.11969029903411865, "p95": 0.20180393755435944 }, { "label": "completed_group_2", "mean": 0.21557100117206573, "median": 0.21654027700424194, "p05": 0.1673085242509842, "p95": 0.2610440254211426 }, { "label": "completed_group_3", "mean": 0.15222224593162537, "median": 0.1510905921459198, "p05": 0.11800260841846466, "p95": 0.18893754482269287 }, { "label": "current_group_partial", "mean": 0.26703086495399475, "median": 0.258539617061615, "p05": 0.19330690801143646, "p95": 0.36521556973457336 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.5552496910095215, "group": 4, "latest_source_mass": 0.31774693727493286, "layer": 15, "max_source_mass": 0.31774693727493286, "mixer_index": 28, "normalized_entropy": 0.9663309649872576, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.2162160575389862, "median": 0.2145705670118332, "p05": 0.17727568745613098, "p95": 0.26878347992897034 }, { "label": "completed_group_1", "mean": 0.17982497811317444, "median": 0.18024154007434845, "p05": 0.15453973412513733, "p95": 0.2030555009841919 }, { "label": "completed_group_2", "mean": 0.1307673156261444, "median": 0.13107559084892273, "p05": 0.11463287472724915, "p95": 0.14630235731601715 }, { "label": "completed_group_3", "mean": 0.1554446816444397, "median": 0.15516987442970276, "p05": 0.13627474009990692, "p95": 0.1754152774810791 }, { "label": "current_group_partial", "mean": 0.31774693727493286, "median": 0.319039523601532, "p05": 0.27606821060180664, "p95": 0.35650834441185 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.5068919658660889, "group": 4, "latest_source_mass": 0.3298327624797821, "layer": 15, "max_source_mass": 0.3298327624797821, "mixer_index": 29, "normalized_entropy": 0.9362846209998111, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.20646697282791138, "median": 0.18775638937950134, "p05": 0.0806753933429718, "p95": 0.4274584949016571 }, { "label": "completed_group_1", "mean": 0.1197461187839508, "median": 0.11989086866378784, "p05": 0.0809398666024208, "p95": 0.15743419528007507 }, { "label": "completed_group_2", "mean": 0.19215428829193115, "median": 0.19620640575885773, "p05": 0.1148109883069992, "p95": 0.2491212636232376 }, { "label": "completed_group_3", "mean": 0.15179988741874695, "median": 0.15212899446487427, "p05": 0.10697861760854721, "p95": 0.19542087614536285 }, { "label": "current_group_partial", "mean": 0.3298327624797821, "median": 0.32837340235710144, "p05": 0.22746729850769043, "p95": 0.4339301884174347 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.5435223579406738, "group": 4, "latest_source_mass": 0.3224271535873413, "layer": 16, "max_source_mass": 0.3224271535873413, "mixer_index": 30, "normalized_entropy": 0.9590443632623663, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.22410975396633148, "median": 0.22760415077209473, "p05": 0.14593973755836487, "p95": 0.30093055963516235 }, { "label": "completed_group_1", "mean": 0.16986176371574402, "median": 0.16963240504264832, "p05": 0.13569067418575287, "p95": 0.20651353895664215 }, { "label": "completed_group_2", "mean": 0.13080403208732605, "median": 0.13075244426727295, "p05": 0.10945598781108856, "p95": 0.15187400579452515 }, { "label": "completed_group_3", "mean": 0.15279728174209595, "median": 0.15036612749099731, "p05": 0.12810495495796204, "p95": 0.18468116223812103 }, { "label": "current_group_partial", "mean": 0.3224271535873413, "median": 0.3164973258972168, "p05": 0.2600978910923004, "p95": 0.40530598163604736 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.4949920177459717, "group": 4, "latest_source_mass": 0.31847020983695984, "layer": 16, "max_source_mass": 0.31847020983695984, "mixer_index": 31, "normalized_entropy": 0.9288907675133354, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.27174079418182373, "median": 0.2585451304912567, "p05": 0.13639721274375916, "p95": 0.4698348045349121 }, { "label": "completed_group_1", "mean": 0.12178683280944824, "median": 0.11714950203895569, "p05": 0.08594125509262085, "p95": 0.1723237782716751 }, { "label": "completed_group_2", "mean": 0.1529826819896698, "median": 0.15223106741905212, "p05": 0.10371473431587219, "p95": 0.203575000166893 }, { "label": "completed_group_3", "mean": 0.1350194811820984, "median": 0.13363957405090332, "p05": 0.08964549005031586, "p95": 0.18468207120895386 }, { "label": "current_group_partial", "mean": 0.31847020983695984, "median": 0.3185216188430786, "p05": 0.21444711089134216, "p95": 0.4301546812057495 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.544264793395996, "group": 5, "latest_source_mass": 0.2861037850379944, "layer": 17, "max_source_mass": 0.2861037850379944, "mixer_index": 32, "normalized_entropy": 0.9595056643474137, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.24016115069389343, "median": 0.23868846893310547, "p05": 0.18435582518577576, "p95": 0.2983453571796417 }, { "label": "completed_group_1", "mean": 0.20730282366275787, "median": 0.1998077780008316, "p05": 0.16255050897598267, "p95": 0.28107208013534546 }, { "label": "completed_group_2", "mean": 0.09692573547363281, "median": 0.0967404842376709, "p05": 0.08098706603050232, "p95": 0.11221462488174438 }, { "label": "completed_group_3", "mean": 0.1695065200328827, "median": 0.1696585714817047, "p05": 0.136396586894989, "p95": 0.20130634307861328 }, { "label": "completed_group_4", "mean": 0.2861037850379944, "median": 0.28883200883865356, "p05": 0.22401469945907593, "p95": 0.3409370183944702 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.686682939529419, "group": 5, "latest_source_mass": 0.12281867861747742, "layer": 17, "max_source_mass": 0.2885604202747345, "mixer_index": 33, "normalized_entropy": 0.9413556721740635, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.16755425930023193, "median": 0.1327776312828064, "p05": 0.0790192037820816, "p95": 0.3667412996292114 }, { "label": "completed_group_1", "mean": 0.10962115228176117, "median": 0.10525335371494293, "p05": 0.07840172946453094, "p95": 0.15405339002609253 }, { "label": "completed_group_2", "mean": 0.1783454418182373, "median": 0.17896884679794312, "p05": 0.10802385956048965, "p95": 0.24048911035060883 }, { "label": "completed_group_3", "mean": 0.13310004770755768, "median": 0.13033035397529602, "p05": 0.0904831737279892, "p95": 0.17934636771678925 }, { "label": "completed_group_4", "mean": 0.2885604202747345, "median": 0.2831825315952301, "p05": 0.16967125236988068, "p95": 0.4189499616622925 }, { "label": "current_group_partial", "mean": 0.12281867861747742, "median": 0.12062528729438782, "p05": 0.07461707293987274, "p95": 0.1782219111919403 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.7070834636688232, "group": 5, "latest_source_mass": 0.2988377809524536, "layer": 18, "max_source_mass": 0.2988377809524536, "mixer_index": 34, "normalized_entropy": 0.9527414214834803, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.181177020072937, "median": 0.18419909477233887, "p05": 0.13624237477779388, "p95": 0.22192169725894928 }, { "label": "completed_group_1", "mean": 0.14557164907455444, "median": 0.144476979970932, "p05": 0.1293315887451172, "p95": 0.16486899554729462 }, { "label": "completed_group_2", "mean": 0.08318156749010086, "median": 0.0820261612534523, "p05": 0.07088497281074524, "p95": 0.0990293100476265 }, { "label": "completed_group_3", "mean": 0.11424209177494049, "median": 0.11267061531543732, "p05": 0.0979941189289093, "p95": 0.1353992223739624 }, { "label": "completed_group_4", "mean": 0.17698991298675537, "median": 0.17597860097885132, "p05": 0.14837387204170227, "p95": 0.21050144731998444 }, { "label": "current_group_partial", "mean": 0.2988377809524536, "median": 0.2989804744720459, "p05": 0.2721370458602905, "p95": 0.32572072744369507 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.7419235706329346, "group": 5, "latest_source_mass": 0.26016175746917725, "layer": 18, "max_source_mass": 0.26016175746917725, "mixer_index": 35, "normalized_entropy": 0.972186055410333, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1532188355922699, "median": 0.1451343297958374, "p05": 0.11514341086149216, "p95": 0.22103679180145264 }, { "label": "completed_group_1", "mean": 0.11420352011919022, "median": 0.11261284351348877, "p05": 0.0928458571434021, "p95": 0.14123988151550293 }, { "label": "completed_group_2", "mean": 0.16597424447536469, "median": 0.1655328869819641, "p05": 0.13871800899505615, "p95": 0.19611573219299316 }, { "label": "completed_group_3", "mean": 0.1250365972518921, "median": 0.12369821965694427, "p05": 0.10375828295946121, "p95": 0.14912742376327515 }, { "label": "completed_group_4", "mean": 0.18140508234500885, "median": 0.1796584278345108, "p05": 0.14511024951934814, "p95": 0.22286373376846313 }, { "label": "current_group_partial", "mean": 0.26016175746917725, "median": 0.26266711950302124, "p05": 0.1889389455318451, "p95": 0.3234381675720215 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.7127439975738525, "group": 5, "latest_source_mass": 0.2907913327217102, "layer": 19, "max_source_mass": 0.2907913327217102, "mixer_index": 36, "normalized_entropy": 0.9559006256078307, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.18731768429279327, "median": 0.19042012095451355, "p05": 0.13879702985286713, "p95": 0.23792000114917755 }, { "label": "completed_group_1", "mean": 0.16715484857559204, "median": 0.1667211651802063, "p05": 0.14574480056762695, "p95": 0.19026139378547668 }, { "label": "completed_group_2", "mean": 0.08496320247650146, "median": 0.08452782034873962, "p05": 0.07410907745361328, "p95": 0.097386933863163 }, { "label": "completed_group_3", "mean": 0.11362075805664062, "median": 0.11248823255300522, "p05": 0.0977015346288681, "p95": 0.13219426572322845 }, { "label": "completed_group_4", "mean": 0.15615218877792358, "median": 0.15540224313735962, "p05": 0.1289147436618805, "p95": 0.18598109483718872 }, { "label": "current_group_partial", "mean": 0.2907913327217102, "median": 0.2900415062904358, "p05": 0.25822532176971436, "p95": 0.32475030422210693 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.6076834201812744, "group": 5, "latest_source_mass": 0.3738824725151062, "layer": 19, "max_source_mass": 0.3738824725151062, "mixer_index": 37, "normalized_entropy": 0.8972652009334232, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.15032559633255005, "median": 0.12544110417366028, "p05": 0.06976518034934998, "p95": 0.35840943455696106 }, { "label": "completed_group_1", "mean": 0.08624927699565887, "median": 0.08602266013622284, "p05": 0.061998721212148666, "p95": 0.11138717085123062 }, { "label": "completed_group_2", "mean": 0.15736976265907288, "median": 0.15802210569381714, "p05": 0.10121746361255646, "p95": 0.20687000453472137 }, { "label": "completed_group_3", "mean": 0.08584291487932205, "median": 0.0827617198228836, "p05": 0.060561805963516235, "p95": 0.11351694166660309 }, { "label": "completed_group_4", "mean": 0.14632999897003174, "median": 0.1426895558834076, "p05": 0.10106660425662994, "p95": 0.20179018378257751 }, { "label": "current_group_partial", "mean": 0.3738824725151062, "median": 0.3799295425415039, "p05": 0.23631010949611664, "p95": 0.4884527325630188 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.709290862083435, "group": 5, "latest_source_mass": 0.2901708781719208, "layer": 20, "max_source_mass": 0.2901708781719208, "mixer_index": 38, "normalized_entropy": 0.9539733939957076, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.18991519510746002, "median": 0.1924753338098526, "p05": 0.14201581478118896, "p95": 0.23709386587142944 }, { "label": "completed_group_1", "mean": 0.17091524600982666, "median": 0.17006167769432068, "p05": 0.15297070145606995, "p95": 0.19153237342834473 }, { "label": "completed_group_2", "mean": 0.07822999358177185, "median": 0.07778562605381012, "p05": 0.06707640737295151, "p95": 0.09089364111423492 }, { "label": "completed_group_3", "mean": 0.116765096783638, "median": 0.11516137421131134, "p05": 0.10162825882434845, "p95": 0.13653048872947693 }, { "label": "completed_group_4", "mean": 0.1540035903453827, "median": 0.15287795662879944, "p05": 0.12593276798725128, "p95": 0.18590238690376282 }, { "label": "current_group_partial", "mean": 0.2901708781719208, "median": 0.29104921221733093, "p05": 0.25643545389175415, "p95": 0.3205650746822357 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.6690887212753296, "group": 5, "latest_source_mass": 0.3342471718788147, "layer": 20, "max_source_mass": 0.3342471718788147, "mixer_index": 39, "normalized_entropy": 0.9315361520005944, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.19029226899147034, "median": 0.19644704461097717, "p05": 0.09456793963909149, "p95": 0.30468082427978516 }, { "label": "completed_group_1", "mean": 0.1100241169333458, "median": 0.10808776319026947, "p05": 0.09035106748342514, "p95": 0.1359245330095291 }, { "label": "completed_group_2", "mean": 0.11471512168645859, "median": 0.11494918912649155, "p05": 0.08799737691879272, "p95": 0.14024731516838074 }, { "label": "completed_group_3", "mean": 0.09999909996986389, "median": 0.09682698547840118, "p05": 0.07921088486909866, "p95": 0.1264391839504242 }, { "label": "completed_group_4", "mean": 0.1507222205400467, "median": 0.1476232260465622, "p05": 0.11397356539964676, "p95": 0.19991227984428406 }, { "label": "current_group_partial", "mean": 0.3342471718788147, "median": 0.33802270889282227, "p05": 0.2548266053199768, "p95": 0.40130624175071716 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.7458693981170654, "group": 6, "latest_source_mass": 0.2361394464969635, "layer": 21, "max_source_mass": 0.2361394464969635, "mixer_index": 40, "normalized_entropy": 0.9743882636597644, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.20528709888458252, "median": 0.2140858769416809, "p05": 0.13168928027153015, "p95": 0.27098023891448975 }, { "label": "completed_group_1", "mean": 0.1722964197397232, "median": 0.17238952219486237, "p05": 0.15053534507751465, "p95": 0.19365043938159943 }, { "label": "completed_group_2", "mean": 0.10435444861650467, "median": 0.10361101478338242, "p05": 0.08888213336467743, "p95": 0.12212536484003067 }, { "label": "completed_group_3", "mean": 0.12075909972190857, "median": 0.1180175170302391, "p05": 0.10358668118715286, "p95": 0.14458929002285004 }, { "label": "completed_group_4", "mean": 0.16116347908973694, "median": 0.15841388702392578, "p05": 0.1337081342935562, "p95": 0.19791515171527863 }, { "label": "completed_group_5", "mean": 0.2361394464969635, "median": 0.23709020018577576, "p05": 0.20446214079856873, "p95": 0.26509594917297363 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.4612125158309937, "group": 6, "latest_source_mass": 0.5543455481529236, "layer": 21, "max_source_mass": 0.5543455481529236, "mixer_index": 41, "normalized_entropy": 0.7509146897354808, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.08934466540813446, "median": 0.07911523431539536, "p05": 0.04488275945186615, "p95": 0.1737232506275177 }, { "label": "completed_group_1", "mean": 0.054379627108573914, "median": 0.05344422906637192, "p05": 0.04221750423312187, "p95": 0.06924191862344742 }, { "label": "completed_group_2", "mean": 0.07567165046930313, "median": 0.07438549399375916, "p05": 0.05811355262994766, "p95": 0.09632256627082825 }, { "label": "completed_group_3", "mean": 0.05758609622716904, "median": 0.05612001568078995, "p05": 0.044792283326387405, "p95": 0.07347827404737473 }, { "label": "completed_group_4", "mean": 0.07884950935840607, "median": 0.07757881283760071, "p05": 0.05758974328637123, "p95": 0.10452556610107422 }, { "label": "completed_group_5", "mean": 0.08982288837432861, "median": 0.08912864327430725, "p05": 0.0663854256272316, "p95": 0.11604885756969452 }, { "label": "current_group_partial", "mean": 0.5543455481529236, "median": 0.5596089363098145, "p05": 0.45455610752105713, "p95": 0.6367484331130981 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.7241814136505127, "group": 6, "latest_source_mass": 0.39484071731567383, "layer": 22, "max_source_mass": 0.39484071731567383, "mixer_index": 42, "normalized_entropy": 0.886053970419732, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.13360010087490082, "median": 0.14072942733764648, "p05": 0.07699814438819885, "p95": 0.17506597936153412 }, { "label": "completed_group_1", "mean": 0.10609637200832367, "median": 0.10586297512054443, "p05": 0.08648421615362167, "p95": 0.12602579593658447 }, { "label": "completed_group_2", "mean": 0.06363911926746368, "median": 0.06324908882379532, "p05": 0.054980404675006866, "p95": 0.07316673547029495 }, { "label": "completed_group_3", "mean": 0.07426095008850098, "median": 0.07362888753414154, "p05": 0.06318790465593338, "p95": 0.08665015548467636 }, { "label": "completed_group_4", "mean": 0.09383076429367065, "median": 0.09294568747282028, "p05": 0.07638302445411682, "p95": 0.11445100605487823 }, { "label": "completed_group_5", "mean": 0.13373197615146637, "median": 0.13327741622924805, "p05": 0.11399447172880173, "p95": 0.15641780197620392 }, { "label": "current_group_partial", "mean": 0.39484071731567383, "median": 0.3879151940345764, "p05": 0.3302172124385834, "p95": 0.4852263331413269 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.3979300260543823, "group": 6, "latest_source_mass": 0.5830056071281433, "layer": 22, "max_source_mass": 0.5830056071281433, "mixer_index": 43, "normalized_entropy": 0.7183939231382495, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.08959861099720001, "median": 0.07544538378715515, "p05": 0.04165133833885193, "p95": 0.20099139213562012 }, { "label": "completed_group_1", "mean": 0.05414731800556183, "median": 0.05336771905422211, "p05": 0.04069708660244942, "p95": 0.07046990841627121 }, { "label": "completed_group_2", "mean": 0.07668881118297577, "median": 0.07607124000787735, "p05": 0.0555989071726799, "p95": 0.09980960935354233 }, { "label": "completed_group_3", "mean": 0.05603286623954773, "median": 0.054573021829128265, "p05": 0.04162490367889404, "p95": 0.0730118378996849 }, { "label": "completed_group_4", "mean": 0.06979228556156158, "median": 0.06884932518005371, "p05": 0.04576495662331581, "p95": 0.09806846082210541 }, { "label": "completed_group_5", "mean": 0.07073447108268738, "median": 0.07023533433675766, "p05": 0.04956750199198723, "p95": 0.09379896521568298 }, { "label": "current_group_partial", "mean": 0.5830056071281433, "median": 0.5862616896629333, "p05": 0.4658474922180176, "p95": 0.6849814057350159 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.7459560632705688, "group": 6, "latest_source_mass": 0.37691548466682434, "layer": 23, "max_source_mass": 0.37691548466682434, "mixer_index": 44, "normalized_entropy": 0.897243926765161, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.14318783581256866, "median": 0.1493408977985382, "p05": 0.08292607218027115, "p95": 0.19464610517024994 }, { "label": "completed_group_1", "mean": 0.1094653457403183, "median": 0.10858799517154694, "p05": 0.09432064741849899, "p95": 0.1279931515455246 }, { "label": "completed_group_2", "mean": 0.06349228322505951, "median": 0.06325754523277283, "p05": 0.05439673364162445, "p95": 0.0737837627530098 }, { "label": "completed_group_3", "mean": 0.07859592139720917, "median": 0.0775163471698761, "p05": 0.06603655964136124, "p95": 0.09357087314128876 }, { "label": "completed_group_4", "mean": 0.09567482024431229, "median": 0.09468592703342438, "p05": 0.07671141624450684, "p95": 0.11774884164333344 }, { "label": "completed_group_5", "mean": 0.13266831636428833, "median": 0.13199329376220703, "p05": 0.11244359612464905, "p95": 0.15504513680934906 }, { "label": "current_group_partial", "mean": 0.37691548466682434, "median": 0.37180638313293457, "p05": 0.31207239627838135, "p95": 0.4595233201980591 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.4055393934249878, "group": 6, "latest_source_mass": 0.5772827863693237, "layer": 23, "max_source_mass": 0.5772827863693237, "mixer_index": 45, "normalized_entropy": 0.7223043644164862, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.10154955089092255, "median": 0.08766824007034302, "p05": 0.04901552200317383, "p95": 0.20287738740444183 }, { "label": "completed_group_1", "mean": 0.05619276314973831, "median": 0.055477917194366455, "p05": 0.041608311235904694, "p95": 0.07371112704277039 }, { "label": "completed_group_2", "mean": 0.07791627198457718, "median": 0.07736755907535553, "p05": 0.055920060724020004, "p95": 0.10158214718103409 }, { "label": "completed_group_3", "mean": 0.053915880620479584, "median": 0.05215863138437271, "p05": 0.03798118978738785, "p95": 0.07156119495630264 }, { "label": "completed_group_4", "mean": 0.06398355215787888, "median": 0.06229078769683838, "p05": 0.040616974234580994, "p95": 0.09270931780338287 }, { "label": "completed_group_5", "mean": 0.06915920227766037, "median": 0.0684593990445137, "p05": 0.046986356377601624, "p95": 0.09408729523420334 }, { "label": "current_group_partial", "mean": 0.5772827863693237, "median": 0.5824689269065857, "p05": 0.4518115520477295, "p95": 0.6926202178001404 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.7853620052337646, "group": 6, "latest_source_mass": 0.3407441973686218, "layer": 24, "max_source_mass": 0.3407441973686218, "mixer_index": 46, "normalized_entropy": 0.9174945750195659, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.15284113585948944, "median": 0.1593906283378601, "p05": 0.08678612112998962, "p95": 0.211342453956604 }, { "label": "completed_group_1", "mean": 0.12206703424453735, "median": 0.12153112143278122, "p05": 0.1060115247964859, "p95": 0.1399766057729721 }, { "label": "completed_group_2", "mean": 0.06508444249629974, "median": 0.06482555717229843, "p05": 0.055350616574287415, "p95": 0.07557765394449234 }, { "label": "completed_group_3", "mean": 0.08347313106060028, "median": 0.08190013468265533, "p05": 0.06966532766819, "p95": 0.10050469636917114 }, { "label": "completed_group_4", "mean": 0.09811970591545105, "median": 0.09664031863212585, "p05": 0.07688035070896149, "p95": 0.12369741499423981 }, { "label": "completed_group_5", "mean": 0.13767032325267792, "median": 0.1373325139284134, "p05": 0.11642646044492722, "p95": 0.1606694608926773 }, { "label": "current_group_partial", "mean": 0.3407441973686218, "median": 0.33805108070373535, "p05": 0.271301805973053, "p95": 0.41781002283096313 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.509842872619629, "group": 6, "latest_source_mass": 0.5227814316749573, "layer": 24, "max_source_mass": 0.5227814316749573, "mixer_index": 47, "normalized_entropy": 0.7759057494780099, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.11788781732320786, "median": 0.10097194463014603, "p05": 0.06265684217214584, "p95": 0.2408325970172882 }, { "label": "completed_group_1", "mean": 0.06822571158409119, "median": 0.0673636943101883, "p05": 0.05209728330373764, "p95": 0.08648915588855743 }, { "label": "completed_group_2", "mean": 0.08556114137172699, "median": 0.08463940769433975, "p05": 0.06234566867351532, "p95": 0.1125786080956459 }, { "label": "completed_group_3", "mean": 0.06067090481519699, "median": 0.058974385261535645, "p05": 0.043267179280519485, "p95": 0.07997901737689972 }, { "label": "completed_group_4", "mean": 0.07228855788707733, "median": 0.06972275674343109, "p05": 0.04689513146877289, "p95": 0.106512151658535 }, { "label": "completed_group_5", "mean": 0.07258446514606476, "median": 0.07082539796829224, "p05": 0.050949547439813614, "p95": 0.09970614314079285 }, { "label": "current_group_partial", "mean": 0.5227814316749573, "median": 0.5290273427963257, "p05": 0.38773614168167114, "p95": 0.6392589807510376 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.839367389678955, "group": 7, "latest_source_mass": 0.29583215713500977, "layer": 25, "max_source_mass": 0.29583215713500977, "mixer_index": 48, "normalized_entropy": 0.9452478525649903, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.15906387567520142, "median": 0.15846745669841766, "p05": 0.09519976377487183, "p95": 0.24517810344696045 }, { "label": "completed_group_1", "mean": 0.13139380514621735, "median": 0.1308426856994629, "p05": 0.11478274315595627, "p95": 0.15000829100608826 }, { "label": "completed_group_2", "mean": 0.07717745751142502, "median": 0.07715500891208649, "p05": 0.06616044044494629, "p95": 0.08791667222976685 }, { "label": "completed_group_3", "mean": 0.09607303142547607, "median": 0.09436552226543427, "p05": 0.08152313530445099, "p95": 0.11384639143943787 }, { "label": "completed_group_4", "mean": 0.1107850894331932, "median": 0.1089654266834259, "p05": 0.08814431726932526, "p95": 0.139452263712883 }, { "label": "completed_group_5", "mean": 0.12967455387115479, "median": 0.1299237608909607, "p05": 0.10832332074642181, "p95": 0.15138982236385345 }, { "label": "completed_group_6", "mean": 0.29583215713500977, "median": 0.29647016525268555, "p05": 0.2276897132396698, "p95": 0.3604946434497833 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.5841834545135498, "group": 7, "latest_source_mass": 0.40767741203308105, "layer": 25, "max_source_mass": 0.40767741203308105, "mixer_index": 49, "normalized_entropy": 0.7618312045616817, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.07896879315376282, "median": 0.06809917092323303, "p05": 0.025932887569069862, "p95": 0.19368338584899902 }, { "label": "completed_group_1", "mean": 0.048752427101135254, "median": 0.04767637699842453, "p05": 0.034443892538547516, "p95": 0.0667153000831604 }, { "label": "completed_group_2", "mean": 0.055707164108753204, "median": 0.0549345538020134, "p05": 0.038696639239788055, "p95": 0.0750277042388916 }, { "label": "completed_group_3", "mean": 0.04139826446771622, "median": 0.03878612071275711, "p05": 0.026673054322600365, "p95": 0.05746251344680786 }, { "label": "completed_group_4", "mean": 0.04449018836021423, "median": 0.04196460545063019, "p05": 0.026162121444940567, "p95": 0.07164917886257172 }, { "label": "completed_group_5", "mean": 0.04285799711942673, "median": 0.04109475016593933, "p05": 0.029393533244729042, "p95": 0.06277710199356079 }, { "label": "completed_group_6", "mean": 0.2801477909088135, "median": 0.2763082981109619, "p05": 0.1455708146095276, "p95": 0.4254714846611023 }, { "label": "current_group_partial", "mean": 0.40767741203308105, "median": 0.4120897948741913, "p05": 0.21399420499801636, "p95": 0.5853031277656555 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.8125314712524414, "group": 7, "latest_source_mass": 0.3700707256793976, "layer": 26, "max_source_mass": 0.3700707256793976, "mixer_index": 50, "normalized_entropy": 0.8716433883436914, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.10889513790607452, "median": 0.11333495378494263, "p05": 0.04850132763385773, "p95": 0.1620434820652008 }, { "label": "completed_group_1", "mean": 0.08656036853790283, "median": 0.08640051633119583, "p05": 0.0650845319032669, "p95": 0.1084962785243988 }, { "label": "completed_group_2", "mean": 0.048123255372047424, "median": 0.04800213500857353, "p05": 0.03576180338859558, "p95": 0.06107594817876816 }, { "label": "completed_group_3", "mean": 0.06561170518398285, "median": 0.06379580497741699, "p05": 0.050968848168849945, "p95": 0.08332888036966324 }, { "label": "completed_group_4", "mean": 0.06917259097099304, "median": 0.06724178791046143, "p05": 0.050207968801259995, "p95": 0.09373342245817184 }, { "label": "completed_group_5", "mean": 0.08286689221858978, "median": 0.08183546364307404, "p05": 0.06535022705793381, "p95": 0.10340933501720428 }, { "label": "completed_group_6", "mean": 0.16869932413101196, "median": 0.16509854793548584, "p05": 0.11059704422950745, "p95": 0.23654375970363617 }, { "label": "current_group_partial", "mean": 0.3700707256793976, "median": 0.3694133758544922, "p05": 0.23441238701343536, "p95": 0.5025027990341187 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.50089693069458, "group": 7, "latest_source_mass": 0.49096518754959106, "layer": 26, "max_source_mass": 0.49096518754959106, "mixer_index": 51, "normalized_entropy": 0.7217788529328457, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.07974998652935028, "median": 0.07402504235506058, "p05": 0.04579866677522659, "p95": 0.1359037309885025 }, { "label": "completed_group_1", "mean": 0.04166398197412491, "median": 0.04038450866937637, "p05": 0.027992289513349533, "p95": 0.05969971418380737 }, { "label": "completed_group_2", "mean": 0.04607272148132324, "median": 0.044438350945711136, "p05": 0.030038591474294662, "p95": 0.06808309257030487 }, { "label": "completed_group_3", "mean": 0.03586764633655548, "median": 0.03317485377192497, "p05": 0.02203454077243805, "p95": 0.05572832003235817 }, { "label": "completed_group_4", "mean": 0.04223340004682541, "median": 0.03930673003196716, "p05": 0.02465122565627098, "p95": 0.06968094408512115 }, { "label": "completed_group_5", "mean": 0.04095568507909775, "median": 0.03919431194663048, "p05": 0.026239857077598572, "p95": 0.06058088690042496 }, { "label": "completed_group_6", "mean": 0.22249138355255127, "median": 0.2125401496887207, "p05": 0.10647299885749817, "p95": 0.3711881935596466 }, { "label": "current_group_partial", "mean": 0.49096518754959106, "median": 0.4988659620285034, "p05": 0.2878011167049408, "p95": 0.6676807403564453 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.8059929609298706, "group": 7, "latest_source_mass": 0.37139976024627686, "layer": 27, "max_source_mass": 0.37139976024627686, "mixer_index": 52, "normalized_entropy": 0.8684990295379666, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.10711923241615295, "median": 0.10972198843955994, "p05": 0.04896187782287598, "p95": 0.16201043128967285 }, { "label": "completed_group_1", "mean": 0.07651707530021667, "median": 0.07666032761335373, "p05": 0.05141128972172737, "p95": 0.10152198374271393 }, { "label": "completed_group_2", "mean": 0.058284737169742584, "median": 0.05812416970729828, "p05": 0.039273619651794434, "p95": 0.07745087146759033 }, { "label": "completed_group_3", "mean": 0.06609587371349335, "median": 0.06383530795574188, "p05": 0.04481930285692215, "p95": 0.09336265176534653 }, { "label": "completed_group_4", "mean": 0.07139429450035095, "median": 0.0694514587521553, "p05": 0.0485839769244194, "p95": 0.10034126043319702 }, { "label": "completed_group_5", "mean": 0.07493151724338531, "median": 0.07319007813930511, "p05": 0.051876723766326904, "p95": 0.10402432829141617 }, { "label": "completed_group_6", "mean": 0.17425751686096191, "median": 0.16937679052352905, "p05": 0.10449051856994629, "p95": 0.2624811828136444 }, { "label": "current_group_partial", "mean": 0.37139976024627686, "median": 0.3732905089855194, "p05": 0.23830299079418182, "p95": 0.5014542937278748 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.5622732639312744, "group": 7, "latest_source_mass": 0.4768889844417572, "layer": 27, "max_source_mass": 0.4768889844417572, "mixer_index": 53, "normalized_entropy": 0.7512946301290215, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.08775827288627625, "median": 0.08504067361354828, "p05": 0.0428270623087883, "p95": 0.144285649061203 }, { "label": "completed_group_1", "mean": 0.047765426337718964, "median": 0.04646281152963638, "p05": 0.031814731657505035, "p95": 0.06807165592908859 }, { "label": "completed_group_2", "mean": 0.053928617388010025, "median": 0.052262187004089355, "p05": 0.035437777638435364, "p95": 0.0784272700548172 }, { "label": "completed_group_3", "mean": 0.04293380305171013, "median": 0.03996016085147858, "p05": 0.02556057646870613, "p95": 0.06623870134353638 }, { "label": "completed_group_4", "mean": 0.04174989461898804, "median": 0.03903091698884964, "p05": 0.025013400241732597, "p95": 0.06869364529848099 }, { "label": "completed_group_5", "mean": 0.04734005779027939, "median": 0.04609068110585213, "p05": 0.029884062707424164, "p95": 0.06911227852106094 }, { "label": "completed_group_6", "mean": 0.20163494348526, "median": 0.19222964346408844, "p05": 0.09963390231132507, "p95": 0.3307305574417114 }, { "label": "current_group_partial", "mean": 0.4768889844417572, "median": 0.48383551836013794, "p05": 0.276212215423584, "p95": 0.6487575173377991 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.8757455348968506, "group": 7, "latest_source_mass": 0.3109493851661682, "layer": 28, "max_source_mass": 0.3109493851661682, "mixer_index": 54, "normalized_entropy": 0.9020429270551009, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.11563669890165329, "median": 0.12087491154670715, "p05": 0.05733206868171692, "p95": 0.1688953936100006 }, { "label": "completed_group_1", "mean": 0.08108697831630707, "median": 0.08185897022485733, "p05": 0.05175650864839554, "p95": 0.10753051191568375 }, { "label": "completed_group_2", "mean": 0.07280858606100082, "median": 0.07357072830200195, "p05": 0.04870978742837906, "p95": 0.0948689877986908 }, { "label": "completed_group_3", "mean": 0.07553227245807648, "median": 0.07362619787454605, "p05": 0.056317005306482315, "p95": 0.09756095707416534 }, { "label": "completed_group_4", "mean": 0.07606014609336853, "median": 0.07456053793430328, "p05": 0.05528895929455757, "p95": 0.1022825762629509 }, { "label": "completed_group_5", "mean": 0.06935927271842957, "median": 0.06843917071819305, "p05": 0.04978285729885101, "p95": 0.09167985618114471 }, { "label": "completed_group_6", "mean": 0.19856664538383484, "median": 0.19508783519268036, "p05": 0.1281973123550415, "p95": 0.27796900272369385 }, { "label": "current_group_partial", "mean": 0.3109493851661682, "median": 0.3105033040046692, "p05": 0.18172834813594818, "p95": 0.43916669487953186 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.6924852132797241, "group": 7, "latest_source_mass": 0.3315581679344177, "layer": 28, "max_source_mass": 0.3315581679344177, "mixer_index": 55, "normalized_entropy": 0.8139133413255193, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.08638577908277512, "median": 0.058608733117580414, "p05": 0.026544082909822464, "p95": 0.2740764319896698 }, { "label": "completed_group_1", "mean": 0.059912845492362976, "median": 0.05901291221380234, "p05": 0.03629481792449951, "p95": 0.08707985281944275 }, { "label": "completed_group_2", "mean": 0.06609142571687698, "median": 0.0647939145565033, "p05": 0.038691695779561996, "p95": 0.09703366458415985 }, { "label": "completed_group_3", "mean": 0.056745611131191254, "median": 0.05418796092271805, "p05": 0.032174739986658096, "p95": 0.08370660990476608 }, { "label": "completed_group_4", "mean": 0.06277773529291153, "median": 0.0600711889564991, "p05": 0.03501531109213829, "p95": 0.10094858705997467 }, { "label": "completed_group_5", "mean": 0.05352145433425903, "median": 0.05179818347096443, "p05": 0.032271772623062134, "p95": 0.08084158599376678 }, { "label": "completed_group_6", "mean": 0.2830069661140442, "median": 0.2759752869606018, "p05": 0.12981083989143372, "p95": 0.46204638481140137 }, { "label": "current_group_partial", "mean": 0.3315581679344177, "median": 0.3204721510410309, "p05": 0.13446375727653503, "p95": 0.5667697787284851 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.8980522155761719, "group": 8, "latest_source_mass": 0.20123262703418732, "layer": 29, "max_source_mass": 0.26602888107299805, "mixer_index": 56, "normalized_entropy": 0.9127701729200177, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.11603894829750061, "median": 0.09991352260112762, "p05": 0.030035395175218582, "p95": 0.2941743731498718 }, { "label": "completed_group_1", "mean": 0.08804173022508621, "median": 0.08880250155925751, "p05": 0.05664174258708954, "p95": 0.11779069900512695 }, { "label": "completed_group_2", "mean": 0.07975024729967117, "median": 0.07984748482704163, "p05": 0.05428915470838547, "p95": 0.10499341040849686 }, { "label": "completed_group_3", "mean": 0.09027000516653061, "median": 0.08898856490850449, "p05": 0.06189651042222977, "p95": 0.11996553838253021 }, { "label": "completed_group_4", "mean": 0.08430293202400208, "median": 0.0819794088602066, "p05": 0.05464346706867218, "p95": 0.12159879505634308 }, { "label": "completed_group_5", "mean": 0.07433462142944336, "median": 0.07287181913852692, "p05": 0.052520088851451874, "p95": 0.1010315865278244 }, { "label": "completed_group_6", "mean": 0.26602888107299805, "median": 0.2639757990837097, "p05": 0.15474572777748108, "p95": 0.3846370279788971 }, { "label": "completed_group_7", "mean": 0.20123262703418732, "median": 0.18140269815921783, "p05": 0.08419936150312424, "p95": 0.3782624900341034 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.7512003183364868, "group": 8, "latest_source_mass": 0.33482813835144043, "layer": 29, "max_source_mass": 0.33482813835144043, "mixer_index": 57, "normalized_entropy": 0.7970056117156376, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.07744461297988892, "median": 0.06970229744911194, "p05": 0.014401955530047417, "p95": 0.17144526541233063 }, { "label": "completed_group_1", "mean": 0.046086348593235016, "median": 0.04394831880927086, "p05": 0.028066720813512802, "p95": 0.0718703418970108 }, { "label": "completed_group_2", "mean": 0.05025433003902435, "median": 0.04861236736178398, "p05": 0.029644077643752098, "p95": 0.07761462777853012 }, { "label": "completed_group_3", "mean": 0.03748488053679466, "median": 0.03434944897890091, "p05": 0.020774798467755318, "p95": 0.05938909202814102 }, { "label": "completed_group_4", "mean": 0.04005061462521553, "median": 0.037498839199543, "p05": 0.019541850313544273, "p95": 0.06922487914562225 }, { "label": "completed_group_5", "mean": 0.041623204946517944, "median": 0.039552345871925354, "p05": 0.022081367671489716, "p95": 0.06821337342262268 }, { "label": "completed_group_6", "mean": 0.198423832654953, "median": 0.18318486213684082, "p05": 0.07548617571592331, "p95": 0.3721975088119507 }, { "label": "completed_group_7", "mean": 0.17380405962467194, "median": 0.1514919549226761, "p05": 0.05777408182621002, "p95": 0.36326974630355835 }, { "label": "current_group_partial", "mean": 0.33482813835144043, "median": 0.33602219820022583, "p05": 0.0795319601893425, "p95": 0.5979493856430054 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.0160694122314453, "group": 8, "latest_source_mass": 0.21711516380310059, "layer": 30, "max_source_mass": 0.21711516380310059, "mixer_index": 58, "normalized_entropy": 0.9175527313077866, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.09385271370410919, "median": 0.09211117774248123, "p05": 0.032377734780311584, "p95": 0.16496515274047852 }, { "label": "completed_group_1", "mean": 0.06208505108952522, "median": 0.06140740588307381, "p05": 0.0407925546169281, "p95": 0.0863042101264 }, { "label": "completed_group_2", "mean": 0.06560768187046051, "median": 0.06516212224960327, "p05": 0.041897669434547424, "p95": 0.09085705131292343 }, { "label": "completed_group_3", "mean": 0.07029052078723907, "median": 0.06780634075403214, "p05": 0.04846132919192314, "p95": 0.09917552024126053 }, { "label": "completed_group_4", "mean": 0.06935097277164459, "median": 0.06708167493343353, "p05": 0.04668639227747917, "p95": 0.09815650433301926 }, { "label": "completed_group_5", "mean": 0.056832700967788696, "median": 0.05595892667770386, "p05": 0.038822539150714874, "p95": 0.0771557092666626 }, { "label": "completed_group_6", "mean": 0.19425815343856812, "median": 0.19145764410495758, "p05": 0.12249615788459778, "p95": 0.27496030926704407 }, { "label": "completed_group_7", "mean": 0.17060703039169312, "median": 0.16317546367645264, "p05": 0.09239593893289566, "p95": 0.2726203501224518 }, { "label": "current_group_partial", "mean": 0.21711516380310059, "median": 0.21198657155036926, "p05": 0.11019522696733475, "p95": 0.3391069769859314 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 1.7837426662445068, "group": 8, "latest_source_mass": 0.29026859998703003, "layer": 30, "max_source_mass": 0.29026859998703003, "mixer_index": 59, "normalized_entropy": 0.8118162725118464, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.08836327493190765, "median": 0.07904411852359772, "p05": 0.02631576545536518, "p95": 0.17534467577934265 }, { "label": "completed_group_1", "mean": 0.04953558370471001, "median": 0.04658038914203644, "p05": 0.025181813165545464, "p95": 0.08318150043487549 }, { "label": "completed_group_2", "mean": 0.04813014343380928, "median": 0.045740917325019836, "p05": 0.026714935898780823, "p95": 0.07843323051929474 }, { "label": "completed_group_3", "mean": 0.0373263955116272, "median": 0.03401809185743332, "p05": 0.01990434154868126, "p95": 0.06290259957313538 }, { "label": "completed_group_4", "mean": 0.037015400826931, "median": 0.03405017405748367, "p05": 0.018646355718374252, "p95": 0.06544196605682373 }, { "label": "completed_group_5", "mean": 0.03390160948038101, "median": 0.03198609501123428, "p05": 0.018189728260040283, "p95": 0.05570102855563164 }, { "label": "completed_group_6", "mean": 0.1840541511774063, "median": 0.17327937483787537, "p05": 0.07966600358486176, "p95": 0.32982927560806274 }, { "label": "completed_group_7", "mean": 0.23140482604503632, "median": 0.22406929731369019, "p05": 0.08831030130386353, "p95": 0.40350067615509033 }, { "label": "current_group_partial", "mean": 0.29026859998703003, "median": 0.2745037376880646, "p05": 0.10280418395996094, "p95": 0.5399085879325867 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 1.9973840713500977, "group": 8, "latest_source_mass": 0.19101998209953308, "layer": 31, "max_source_mass": 0.20831164717674255, "mixer_index": 60, "normalized_entropy": 0.9090486661912383, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.10261696577072144, "median": 0.10389091819524765, "p05": 0.01371697522699833, "p95": 0.21163305640220642 }, { "label": "completed_group_1", "mean": 0.06039813905954361, "median": 0.06085003539919853, "p05": 0.026107093319296837, "p95": 0.09348798543214798 }, { "label": "completed_group_2", "mean": 0.06671924144029617, "median": 0.06656201928853989, "p05": 0.034684643149375916, "p95": 0.09819839894771576 }, { "label": "completed_group_3", "mean": 0.08377155661582947, "median": 0.0804532840847969, "p05": 0.05448959022760391, "p95": 0.12039553374052048 }, { "label": "completed_group_4", "mean": 0.07002593576908112, "median": 0.06737451255321503, "p05": 0.044841229915618896, "p95": 0.10655929893255234 }, { "label": "completed_group_5", "mean": 0.05768897384405136, "median": 0.056077782064676285, "p05": 0.039495617151260376, "p95": 0.08242568373680115 }, { "label": "completed_group_6", "mean": 0.20831164717674255, "median": 0.2021285891532898, "p05": 0.11652229726314545, "p95": 0.31706833839416504 }, { "label": "completed_group_7", "mean": 0.1594475358724594, "median": 0.1546279788017273, "p05": 0.06593266129493713, "p95": 0.2735573351383209 }, { "label": "current_group_partial", "mean": 0.19101998209953308, "median": 0.1713225245475769, "p05": 0.06408411264419556, "p95": 0.3850628137588501 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 1.969987154006958, "group": 8, "latest_source_mass": 0.16864418983459473, "layer": 31, "max_source_mass": 0.24503237009048462, "mixer_index": 61, "normalized_entropy": 0.8965797917640489, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.08120635151863098, "median": 0.061164144426584244, "p05": 0.0259944386780262, "p95": 0.23622018098831177 }, { "label": "completed_group_1", "mean": 0.07894632965326309, "median": 0.07658770680427551, "p05": 0.0496862456202507, "p95": 0.11596276611089706 }, { "label": "completed_group_2", "mean": 0.08615730702877045, "median": 0.08467075228691101, "p05": 0.050409119576215744, "p95": 0.12811316549777985 }, { "label": "completed_group_3", "mean": 0.07609779387712479, "median": 0.0722871869802475, "p05": 0.04099677503108978, "p95": 0.11950768530368805 }, { "label": "completed_group_4", "mean": 0.058728985488414764, "median": 0.056080836802721024, "p05": 0.03323862701654434, "p95": 0.09344407916069031 }, { "label": "completed_group_5", "mean": 0.05076971277594566, "median": 0.04946017265319824, "p05": 0.03047419898211956, "p95": 0.07563400268554688 }, { "label": "completed_group_6", "mean": 0.24503237009048462, "median": 0.24110376834869385, "p05": 0.10613486170768738, "p95": 0.3991503119468689 }, { "label": "completed_group_7", "mean": 0.1544169783592224, "median": 0.1400495171546936, "p05": 0.06199025735259056, "p95": 0.2995939254760742 }, { "label": "current_group_partial", "mean": 0.16864418983459473, "median": 0.15222924947738647, "p05": 0.0543956533074379, "p95": 0.3329208791255951 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.071648359298706, "group": 8, "latest_source_mass": 0.11520501971244812, "layer": 32, "max_source_mass": 0.20955611765384674, "mixer_index": 62, "normalized_entropy": 0.9428478002054053, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.10569609701633453, "median": 0.08689188957214355, "p05": 0.03173588588833809, "p95": 0.24571751058101654 }, { "label": "completed_group_1", "mean": 0.09922641515731812, "median": 0.09795533865690231, "p05": 0.06407676637172699, "p95": 0.14120742678642273 }, { "label": "completed_group_2", "mean": 0.08180510997772217, "median": 0.080217644572258, "p05": 0.054607540369033813, "p95": 0.11442367732524872 }, { "label": "completed_group_3", "mean": 0.10646799206733704, "median": 0.10510502755641937, "p05": 0.07415887713432312, "p95": 0.14216329157352448 }, { "label": "completed_group_4", "mean": 0.0826798751950264, "median": 0.08022409677505493, "p05": 0.05491747707128525, "p95": 0.11860857903957367 }, { "label": "completed_group_5", "mean": 0.06615597754716873, "median": 0.06485182791948318, "p05": 0.04477996751666069, "p95": 0.09238749742507935 }, { "label": "completed_group_6", "mean": 0.20955611765384674, "median": 0.206287682056427, "p05": 0.129981130361557, "p95": 0.2996031939983368 }, { "label": "completed_group_7", "mean": 0.13320739567279816, "median": 0.12465103715658188, "p05": 0.06303662061691284, "p95": 0.23065781593322754 }, { "label": "current_group_partial", "mean": 0.11520501971244812, "median": 0.10386130958795547, "p05": 0.03912060335278511, "p95": 0.22735844552516937 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 1.9781701564788818, "group": 8, "latest_source_mass": 0.16112206876277924, "layer": 32, "max_source_mass": 0.22251944243907928, "mixer_index": 63, "normalized_entropy": 0.9003040366848136, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.09446848928928375, "median": 0.074150450527668, "p05": 0.042592622339725494, "p95": 0.19443637132644653 }, { "label": "completed_group_1", "mean": 0.07056106626987457, "median": 0.06868506222963333, "p05": 0.04818308353424072, "p95": 0.0982733890414238 }, { "label": "completed_group_2", "mean": 0.08033448457717896, "median": 0.08047192543745041, "p05": 0.048296794295310974, "p95": 0.11253741383552551 }, { "label": "completed_group_3", "mean": 0.0658978745341301, "median": 0.06256154924631119, "p05": 0.04223886504769325, "p95": 0.0967884361743927 }, { "label": "completed_group_4", "mean": 0.058234501630067825, "median": 0.0563921295106411, "p05": 0.03529864549636841, "p95": 0.08846980333328247 }, { "label": "completed_group_5", "mean": 0.04316519945859909, "median": 0.042268507182598114, "p05": 0.029443470761179924, "p95": 0.05996086448431015 }, { "label": "completed_group_6", "mean": 0.2036968618631363, "median": 0.19645483791828156, "p05": 0.09398645162582397, "p95": 0.3395216464996338 }, { "label": "completed_group_7", "mean": 0.22251944243907928, "median": 0.2130982130765915, "p05": 0.11867167800664902, "p95": 0.35465875267982483 }, { "label": "current_group_partial", "mean": 0.16112206876277924, "median": 0.14023703336715698, "p05": 0.06008770316839218, "p95": 0.3247341811656952 } ], "sources": 9 }, { "branch": "output", "entropy_mean": 1.6036605834960938, "group": null, "latest_source_mass": 0.31639498472213745, "layer": null, "max_source_mass": 0.3412485122680664, "mixer_index": null, "normalized_entropy": 0.7298573846467136, "offset": null, "source_summaries": [ { "label": "embedding", "mean": 0.09759747982025146, "median": 0.09915056824684143, "p05": 0.025461740791797638, "p95": 0.16398483514785767 }, { "label": "completed_group_1", "mean": 0.04215940088033676, "median": 0.04050712287425995, "p05": 0.025720715522766113, "p95": 0.06398092210292816 }, { "label": "completed_group_2", "mean": 0.041145481169223785, "median": 0.03881259262561798, "p05": 0.023510465398430824, "p95": 0.0662468671798706 }, { "label": "completed_group_3", "mean": 0.029554028064012527, "median": 0.027420252561569214, "p05": 0.01627083122730255, "p95": 0.04924973472952843 }, { "label": "completed_group_4", "mean": 0.0289996936917305, "median": 0.027226522564888, "p05": 0.01560397818684578, "p95": 0.048351090401411057 }, { "label": "completed_group_5", "mean": 0.03647579252719879, "median": 0.0347352996468544, "p05": 0.020744025707244873, "p95": 0.058257561177015305 }, { "label": "completed_group_6", "mean": 0.06642461568117142, "median": 0.057809896767139435, "p05": 0.022837381809949875, "p95": 0.14078694581985474 }, { "label": "completed_group_7", "mean": 0.3412485122680664, "median": 0.3336990475654602, "p05": 0.0819513276219368, "p95": 0.6423859596252441 }, { "label": "completed_group_8", "mean": 0.31639498472213745, "median": 0.30713334679603577, "p05": 0.05841828137636185, "p95": 0.6156357526779175 } ], "sources": 9 } ], "mode": "uniform_group_7_mlp_only", "optimizer_state_after": "ed03e6fbd4a12d8b063dcb22e0437754285f54d585374cd52fbd534f05d24637", "optimizer_state_before": "ed03e6fbd4a12d8b063dcb22e0437754285f54d585374cd52fbd534f05d24637", "optimizer_state_unchanged": true, "positions": { "attention_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "5e6ecbcd78543fc42d4098a51169dca446d404ce384015643e84d2c78433c949", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.0933690071105957, 0.24537226557731628, 0.07698935270309448, 0.0523640513420105, 0.09380833059549332, 0.03647654876112938, 0.039120711386203766, 0.06371714919805527, 0.05156813934445381, 0.03728010877966881, 0.03983496502041817, 0.040284499526023865, 0.046871740370988846, 0.04314620420336723, 0.04033593460917473, 0.04471700266003609, 0.08397205173969269, 0.0528092160820961, 0.04443390294909477, 0.07560808956623077, 0.03708760067820549, 0.03599191829562187, 0.033304885029792786, 0.04061548039317131, 0.025487789884209633, 0.021565141156315804, 0.02342291921377182, 0.017244853079319, 0.024166326969861984, 0.016819356009364128, 0.021590016782283783, 0.026201345026493073 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.4685618111798249e-05, "first_to_last_ratio": 0.9343140631802743, "imbalance_abs_log_ratio": 0.06794264119303642, "last_quartile_mean": 1.5718074564574636e-05, "mean": 1.5263751805605352e-05, "normalized": [ 0.895061017015498, 1.0413843148265105, 0.9289500771054747, 0.8623770689174891, 1.0959503557764507, 1.0227114843670337, 0.9773103126678518, 0.8732453617344464, 0.8344039914843494, 0.7634705335770949, 0.7254353519673601, 0.7001398576038405, 0.8360865602076716, 0.7831866424478136, 0.7468200866093059, 0.709090636996149, 0.794634461555692, 0.7530570550906538, 0.7519065827870238, 0.7443390157448534, 2.2459905470620596, 1.9126981010136825, 1.5395191591607402, 1.2241128988985166, 1.2339986902749025, 1.1949361395197777, 1.1469280528756964, 1.0833822740944812, 1.3716157466226564, 0.9787689599639396, 0.6473123359413984, 0.5811763260895855 ], "peak_layer": 21, "peak_normalized": 2.2459905470620596, "population_cv": 0.35579243487797, "rest_mean": 1.3479408254271322e-05, "spike_contrast": 1.84720326835706, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.489920698280912e-05, "top_five_layers": [ 21, 22, 23, 29, 25 ], "top_five_spike_overlap": 4 }, "values": [ 1.366198921459727e-05, 1.5895431715762243e-05, 1.4179263416735921e-05, 1.3163109542801976e-05, 1.6728314221836627e-05, 1.5610414266120642e-05, 1.4917422049620654e-05, 1.3329000466910657e-05, 1.2736135431623552e-05, 1.1653424735413864e-05, 1.1072865163441747e-05, 1.0686761015676893e-05, 1.2761817743012216e-05, 1.1954366527788807e-05, 1.1399276445445139e-05, 1.0823383490787819e-05, 1.212910319736693e-05, 1.1494475984363817e-05, 1.1476915460661985e-05, 1.1361405995558016e-05, 3.428224226809107e-05, 2.9194949092925526e-05, 2.3498838345403783e-05, 1.8684555470827036e-05, 1.8835449736798182e-05, 1.8239208657178096e-05, 1.750642513798084e-05, 1.653647814237047e-05, 2.0936002329108305e-05, 1.4939686479920056e-05, 9.880414836516138e-06, 8.870931196724996e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 5.822437196911778e-05, "first_to_last_ratio": 0.9289066147564641, "imbalance_abs_log_ratio": 0.07374706754989334, "last_quartile_mean": 6.268054403335555e-05, "mean": 6.0844860968245484e-05, "normalized": [ 0.8949416625809286, 1.0229002137431409, 0.9240110600981873, 0.8631094397068032, 1.0908980764443927, 1.0193614199937575, 0.9710020509324288, 0.86922911461376, 0.833588799050515, 0.7628096953455408, 0.7274045380260752, 0.7023956112673467, 0.8420542566623269, 0.7882567112098301, 0.7515003954889087, 0.7121538114138023, 0.8001029347962203, 0.7563086730501211, 0.7544308151903758, 0.7490057928326762, 2.2439908147701195, 1.9218274886739521, 1.5396942065023351, 1.2176632578157096, 1.2267025900331137, 1.186392054469014, 1.1411414618447724, 1.0815593105923973, 1.3741845370424, 0.9846589272257527, 0.6560149077949806, 0.5907053707883151 ], "peak_layer": 21, "peak_normalized": 2.2439908147701195, "population_cv": 0.3544913326246491, "rest_mean": 5.374656797761822e-05, "spike_contrast": 1.8452460659242949, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.917564311763271e-05, "top_five_layers": [ 21, 22, 23, 29, 25 ], "top_five_spike_overlap": 4 }, "values": [ 5.445260103442706e-05, 6.223822128959e-05, 5.622132448479533e-05, 5.25157738593407e-05, 6.637554179178551e-05, 6.202290387591347e-05, 5.908048478886485e-05, 5.288812462822534e-05, 5.0719594582915306e-05, 4.641304985852912e-05, 4.425882798386738e-05, 4.273716331226751e-05, 5.123467417433858e-05, 4.796137000084855e-05, 4.5724937081104144e-05, 4.3330899643478915e-05, 4.8682151827961206e-05, 4.6017496060812846e-05, 4.5903238060418516e-05, 4.5573153329314664e-05, 0.00013653530913870782, 0.00011693332635331899, 9.368247992824763e-05, 7.40885516279377e-05, 7.463854853995144e-05, 7.218585960799828e-05, 6.943259359104559e-05, 6.580732588190585e-05, 8.361206710105762e-05, 5.991143552819267e-05, 3.991513585788198e-05, 3.594138615881093e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.0516339554451406, "first_to_last_ratio": 0.9289066190158382, "imbalance_abs_log_ratio": 0.07374706296453029, "last_quartile_mean": 0.05558573314920068, "mean": 0.05395783099811524, "normalized": [ 0.8949416952731531, 1.0229002048070763, 0.9240110360897064, 0.8631094118086762, 1.0908980467392606, 1.0193614510060824, 0.9710021185398164, 0.8692291167830459, 0.8335888205074324, 0.7628096710820175, 0.727404527675262, 0.7023956127316612, 0.8420542547455482, 0.7882567187636609, 0.7515003784113916, 0.7121537663815494, 0.80010294151682, 0.7563086539248226, 0.7544308139717784, 0.749005797243577, 2.2439909204567265, 1.9218274857565165, 1.5396941319911421, 1.2176632555728486, 1.2267026257471993, 1.1863920664610461, 1.1411414328280356, 1.0815593848190204, 1.3741844847396258, 0.9846588664934841, 0.6560149197453131, 0.5907053873867028 ], "peak_layer": 21, "peak_normalized": 2.2439909204567265, "population_cv": 0.35449134068886035, "rest_mean": 0.04766299368606673, "spike_contrast": 1.845246084676538, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.08794995248317719, "top_five_layers": [ 21, 22, 23, 29, 25 ], "top_five_spike_overlap": 4 }, "values": [ 0.048289112746715546, 0.055193476378917694, 0.04985763132572174, 0.0465715117752552, 0.05886249244213104, 0.05500253289937973, 0.052393168210983276, 0.04690171778202057, 0.04497864469885826, 0.041159555315971375, 0.03924917057156563, 0.037899743765592575, 0.04543542116880417, 0.04253262281417847, 0.04054933041334152, 0.038426272571086884, 0.04317181929945946, 0.040808774530887604, 0.04070745036005974, 0.040414728224277496, 0.12108088284730911, 0.10369764268398285, 0.08307855576276779, 0.06570246815681458, 0.0661902129650116, 0.06401514261960983, 0.061573516577482224, 0.05835859850049019, 0.0741480141878128, 0.05313005670905113, 0.03539714217185974, 0.0318731814622879 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 2.3600840677318047e-06, "first_to_last_ratio": 0.5413484929200453, "imbalance_abs_log_ratio": 0.6136920431580876, "last_quartile_mean": 4.359639120821157e-06, "mean": 3.3537639936298547e-06, "normalized": [ 0.7862244144533114, 0.702849013528531, 0.6663388688669981, 0.651590811100317, 0.7209207300517917, 0.7063940958633373, 0.697144136052399, 0.6982341692093892, 0.649554066722764, 0.6239434400723638, 0.6187273080265554, 0.6237127961614246, 0.669663002199526, 0.6705705948026252, 0.6781541828654816, 0.6899791917726259, 0.6887646841240178, 0.7135149723764701, 0.7336622807084913, 0.7558981977895731, 2.2739718772352275, 2.1438115589756137, 1.8607548023256535, 1.5762268602355463, 1.650885928105121, 1.5996313188948021, 1.504528739504819, 1.3664508673077789, 1.6238106922812876, 1.1718599487986365, 0.7902574964562366, 0.691968953131284 ], "peak_layer": 21, "peak_normalized": 2.2739718772352275, "population_cv": 0.49134482122551276, "rest_mean": 2.7941013943038543e-06, "spike_contrast": 2.281929368414639, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.375942029990256e-06, "top_five_layers": [ 21, 22, 23, 25, 29 ], "top_five_spike_overlap": 4 }, "values": [ 2.6368111321062315e-06, 2.35718971453025e-06, 2.2347433059621835e-06, 2.1852818008483155e-06, 2.417797986709047e-06, 2.3690790840191767e-06, 2.3380569018627284e-06, 2.3417126158165047e-06, 2.17845104089065e-06, 2.092559043376241e-06, 2.0750653675349895e-06, 2.091785518132383e-06, 2.2458916646428406e-06, 2.248935516035999e-06, 2.2743690806237282e-06, 2.3140273697208613e-06, 2.3099541976989713e-06, 2.3929608232720057e-06, 2.4605301405244973e-06, 2.5351041585963685e-06, 7.626365004398394e-06, 7.189838015619898e-06, 6.2405324570136145e-06, 5.286292889650213e-06, 5.5366817832691595e-06, 5.364785920392023e-06, 5.045834313932573e-06, 4.582753717841115e-06, 5.44587783224415e-06, 3.930141701857792e-06, 2.6503371373109985e-06, 2.3207005597214447e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 4.4239825911063235e-05, "first_to_last_ratio": 1.1136916966716648, "imbalance_abs_log_ratio": 0.10768034976522613, "last_quartile_mean": 3.97235842228838e-05, "mean": 4.33480432207034e-05, "normalized": [ 0.9700329145857894, 1.093936568464863, 0.9932431760909444, 0.9300907199892268, 1.1680987083804137, 1.0878915441448065, 1.0182345750705133, 0.9030527291776335, 0.8761982820524437, 0.7982776508465873, 0.7546472100742688, 0.7221274147320358, 0.8939446989487194, 0.8268322479485324, 0.7794580866626835, 0.7297624796019788, 0.8552712805071861, 0.8019710164392936, 0.7991526515207729, 0.7951214883342054, 2.3062483240515292, 1.9277913136715141, 1.5041573789758065, 1.1333613662499586, 1.2016228326184821, 1.1258682183318713, 1.0645308892761238, 1.0012423070384453, 1.0908629040977642, 0.7755241923981744, 0.5600037045485814, 0.5114411251688514 ], "peak_layer": 21, "peak_normalized": 2.3062483240515292, "population_cv": 0.3513283104552832, "rest_mean": 3.841410277042693e-05, "spike_contrast": 1.8220215130490858, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.999132165219635e-05, "top_five_layers": [ 21, 22, 23, 25, 5 ], "top_five_spike_overlap": 4 }, "values": [ 4.204902870696969e-05, 4.742000965052284e-05, 4.305514812585898e-05, 4.031761272926815e-05, 5.0634793296922e-05, 4.7157969675026834e-05, 4.413847636897117e-05, 3.914556873496622e-05, 3.79814810003154e-05, 3.460377411101945e-05, 3.271247987868264e-05, 3.1302810384659097e-05, 3.875075344694778e-05, 3.584156002034433e-05, 3.378798282938078e-05, 3.1633775506634265e-05, 3.707433643285185e-05, 3.476387428236194e-05, 3.4641703678062186e-05, 3.446696064202115e-05, 9.997135202866048e-05, 8.356598118552938e-05, 6.52022790745832e-05, 4.912899748887867e-05, 5.208799848333001e-05, 4.880418418906629e-05, 4.614533099811524e-05, 4.340189479989931e-05, 4.7286772314691916e-05, 3.3617456210777164e-05, 2.4275064788525924e-05, 2.216997199866455e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 3.203852793376427e-05, "first_to_last_ratio": 1.289265539426739, "imbalance_abs_log_ratio": 0.2540727069511347, "last_quartile_mean": 2.4850216618688137e-05, "mean": 3.0209900131694667e-05, "normalized": [ 1.0267082094898252, 1.142818040734902, 1.0412722151638483, 0.9754175965191436, 1.2277842138072939, 1.1371921004580787, 1.0286349838472286, 0.9044186107635691, 0.900731125913691, 0.811583431003843, 0.76349854908199, 0.7153190143698296, 0.9294742038721472, 0.853992878632567, 0.7981426143567888, 0.7371550879505838, 0.9029181352327412, 0.8396322078649486, 0.8419022491164632, 0.8395372540161479, 2.4690062336767085, 1.9889176473655712, 1.4974243632095656, 1.045837466156921, 1.1860329402567191, 1.0895439339866708, 1.0147982150697281, 0.9539131087629588, 0.8411049258026901, 0.5950952188132032, 0.46396046134079333, 0.43623276336283945 ], "peak_layer": 21, "peak_normalized": 2.4690062336767085, "population_cv": 0.38459517700918705, "rest_mean": 2.6643768385973225e-05, "spike_contrast": 1.8566071751559237, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.9467011558590455e-05, "top_five_layers": [ 21, 22, 23, 5, 25 ], "top_five_spike_overlap": 4 }, "values": [ 3.101675247307867e-05, 3.4524418879300356e-05, 3.145672963000834e-05, 2.9467268177540973e-05, 3.70912384823896e-05, 3.4354459785390645e-05, 3.1074960133992136e-05, 2.7322395908413455e-05, 2.72109973593615e-05, 2.4517854399164207e-05, 2.3065214918460697e-05, 2.1609715986414813e-05, 2.8079322873963974e-05, 2.5799039576668292e-05, 2.411180867056828e-05, 2.2269381588557735e-05, 2.7277066692477092e-05, 2.5365205146954395e-05, 2.5433782866457477e-05, 2.5362336600665003e-05, 7.458843174390495e-05, 6.0085003497079015e-05, 4.523704046732746e-05, 3.159464540658519e-05, 3.5829936678055674e-05, 3.291501343483105e-05, 3.065695273107849e-05, 2.8817619750043377e-05, 2.5409695808775723e-05, 1.7977767129195854e-05, 1.4016199202160351e-05, 1.3178548215364572e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00012353743204585044, "first_to_last_ratio": 0.9287412692418178, "imbalance_abs_log_ratio": 0.0739250835415215, "last_quartile_mean": 0.0001330159821009147, "mean": 0.00012799522005479957, "normalized": [ 0.9122126856898034, 1.0003892160944914, 0.9265141712888798, 0.8741159405786518, 1.0943900270068017, 1.0291665894144637, 0.9949566013689213, 0.8896326133218404, 0.8565510661238337, 0.7933198942301297, 0.7515175019491048, 0.736075751629827, 0.8679805444215124, 0.8095005228999325, 0.7700179971741407, 0.7275115728596298, 0.8140361149780735, 0.7711424019882183, 0.7739998568309505, 0.7571441305083678, 2.0791554733576127, 1.8111919566628207, 1.4593503420416671, 1.1863176810146927, 1.280082924002355, 1.2475756873117612, 1.1967375824924513, 1.1413491057541685, 1.351356659975964, 0.9238618720060063, 0.613873071661332, 0.5589724433615937 ], "peak_layer": 21, "peak_normalized": 2.0791554733576127, "population_cv": 0.3241858818077022, "rest_mean": 0.00011464532629664366, "spike_contrast": 1.7452490460110377, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00020008464634884148, "top_five_layers": [ 21, 22, 23, 29, 25 ], "top_five_spike_overlap": 4 }, "values": [ 0.0001167588634416461, 0.00012804503785446286, 0.00011858938523801044, 0.00011188266216777265, 0.00014007669233251363, 0.00013172840408515185, 0.00012734968913719058, 0.00011386872211005539, 0.00010963444219669327, 0.00010154115443583578, 9.619064803700894e-05, 9.42141778068617e-05, 0.00011109736078651622, 0.00010361219756305218, 9.855862299446017e-05, 9.311800386058167e-05, 0.00010419273166917264, 9.870254143606871e-05, 9.906828199746087e-05, 9.691082959761843e-05, 0.00026612196234054863, 0.00023182391305454075, 0.0001867898681666702, 0.00015184299263637513, 0.00016384449554607272, 0.0001596837246324867, 0.00015317669021897018, 0.0001460872299503535, 0.0001729671930661425, 0.00011824990360764787, 7.857281889300793e-05, 7.154580089263618e-05 ] } } }, "mlp_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "c0b8018c349b1bd09e5d17858eb312a2323da0e92de9651e2bb08ae4adf115c7", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.13583384454250336, 0.0415157787501812, 0.018140379339456558, 0.039518751204013824, 0.022877270355820656, 0.015532293356955051, 0.02314678393304348, 0.05432315170764923, 0.03743875026702881, 0.04006272554397583, 0.045583780854940414, 0.05995568633079529, 0.038651175796985626, 0.04559725895524025, 0.05632574483752251, 0.07952121645212173, 0.042337141931056976, 0.05044656619429588, 0.04418695345520973, 0.06310147047042847, 0.029010163620114326, 0.04586305841803551, 0.06033914536237717, 0.07656749337911606, 0.045884523540735245, 0.051604680716991425, 0.06502768397331238, 0.06792232394218445, 0.060306549072265625, 0.08025261014699936, 0.09663844108581543, 0.12230555713176727 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.4556978840118973e-05, "first_to_last_ratio": 1.0521172327964308, "imbalance_abs_log_ratio": 0.05080454612618417, "last_quartile_mean": 1.3835890513291815e-05, "mean": 1.4168472262099385e-05, "normalized": [ 0.9626602218456324, 1.1208433471986854, 1.0015833461716221, 0.9273098007497561, 1.1757692702716582, 1.0895604169876238, 1.0279865069583527, 0.9136510809839264, 0.8429412519899347, 0.7964256277923278, 0.761633479689827, 0.737990422636471, 0.8789091947825194, 0.818438864068197, 0.7730429954295691, 0.7408952780773379, 0.8509649257946901, 0.7987762543008268, 0.7947856005751553, 0.8213145765932317, 2.1916325168181583, 1.706846659477225, 1.3663122015097828, 1.0875130950546912, 1.3033613208031634, 1.2491937554562729, 1.1759083089405844, 1.1148926719046885, 1.0543459533713346, 0.7015882889128537, 0.6285119781235557, 0.5844107867303446 ], "peak_layer": 21, "peak_normalized": 2.1916325168181583, "population_cv": 0.3174845739642227, "rest_mean": 1.2774889775620843e-05, "spike_contrast": 1.698160850709119, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.1693817689083517e-05, "top_five_layers": [ 21, 22, 23, 25, 26 ], "top_five_spike_overlap": 4 }, "values": [ 1.3639424651046284e-05, 1.5880637874943204e-05, 1.4190905858413316e-05, 1.3138563190295827e-05, 1.6658854292472824e-05, 1.543740654597059e-05, 1.4564998309651855e-05, 1.2945039998157881e-05, 1.194318974739872e-05, 1.1284134416200686e-05, 1.0791182830871549e-05, 1.0456196832819842e-05, 1.2452800547180232e-05, 1.1596028343774378e-05, 1.0952838238154072e-05, 1.0497354196559172e-05, 1.2056872947141528e-05, 1.1317439202684909e-05, 1.126089773606509e-05, 1.1636772796919104e-05, 3.105208452325314e-05, 2.4183409550460055e-05, 1.9358556528459303e-05, 1.5408399121952243e-05, 1.846663872129284e-05, 1.7699167074169964e-05, 1.6660824257996865e-05, 1.579632589709945e-05, 1.4938471394998487e-05, 9.940434210875537e-06, 8.905054528440814e-06, 8.280208021460567e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 5.772661779701593e-05, "first_to_last_ratio": 1.0446007586449662, "imbalance_abs_log_ratio": 0.04363476327383373, "last_quartile_mean": 5.5261895340663614e-05, "mean": 5.650842990689853e-05, "normalized": [ 0.961481809059367, 1.1002371201938812, 0.9957655777536307, 0.9275574061753838, 1.1697722799630106, 1.0860927085167182, 1.0216976644155618, 0.9098564833091597, 0.8414681781699731, 0.7957458639766367, 0.7638988555688521, 0.7409044009246765, 0.8860241951545947, 0.8249574372698293, 0.7780181592800512, 0.7440241622647688, 0.8560566145682945, 0.800143298887283, 0.795546352216325, 0.8260919310549901, 2.192019937591978, 1.7189377030443285, 1.3575308082689208, 1.0826451916516995, 1.2956770122202925, 1.240378197911393, 1.169547080822136, 1.1132905585602204, 1.0605320266128506, 0.7105136545427877, 0.6386547203340158, 0.5949326097163893 ], "peak_layer": 21, "peak_normalized": 2.192019937591978, "population_cv": 0.3154281669963968, "rest_mean": 5.096890715692468e-05, "spike_contrast": 1.69557986579306, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 8.642185275675729e-05, "top_five_layers": [ 21, 22, 23, 25, 26 ], "top_five_spike_overlap": 4 }, "values": [ 5.433182741398923e-05, 6.217267218744382e-05, 5.626914935419336e-05, 5.241481267148629e-05, 6.610199488932267e-05, 6.137339369161054e-05, 5.773453085566871e-05, 5.141456131241284e-05, 4.755004556500353e-05, 4.496634937822819e-05, 4.316672493587248e-05, 4.1867344407364726e-05, 5.00678361277096e-05, 4.661704952013679e-05, 4.396458461997099e-05, 4.204363722237758e-05, 4.837441520066932e-05, 4.521484152064659e-05, 4.495507528190501e-05, 4.668115798267536e-05, 0.00012386760499794036, 9.713447070680559e-05, 7.671193452551961e-05, 6.117857992649078e-05, 7.32166736270301e-05, 7.009182445472106e-05, 6.608926923945546e-05, 6.291030149441212e-05, 5.992899968987331e-05, 4.015001104562543e-05, 3.608937549870461e-05, 3.36187076754868e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.051192541141062975, "first_to_last_ratio": 1.044600734262081, "imbalance_abs_log_ratio": 0.043634739932011384, "last_quartile_mean": 0.04900680179707706, "mean": 0.050112241005990654, "normalized": [ 0.9614817848514211, 1.100237030254126, 0.995765560550454, 0.9275574378714293, 1.1697723258227146, 1.0860926366774486, 1.021697654994183, 0.9098564469410632, 0.8414682010384115, 0.7957458944192425, 0.763898874455773, 0.7409043571085546, 0.8860242042091758, 0.8249574506914098, 0.7780181414746354, 0.7440242132576173, 0.8560566210347027, 0.8001432650536664, 0.7955463687352908, 0.8260919370784391, 2.192019940209548, 1.7189377043786946, 1.3575308264515957, 1.0826452432096485, 1.2956769723594332, 1.2403782506387964, 1.1695470788695201, 1.1132905743947727, 1.060532013859169, 0.7105136367533613, 0.6386547292983057, 0.5949326230573952 ], "peak_layer": 21, "peak_normalized": 2.192019940209548, "population_cv": 0.3154281663296825, "rest_mean": 0.045199736745821104, "spike_contrast": 1.6955798756503129, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.07663976401090622, "top_five_layers": [ 21, 22, 23, 25, 26 ], "top_five_spike_overlap": 4 }, "values": [ 0.04818200692534447, 0.055135343223810196, 0.04990004375576973, 0.04648198187351227, 0.0586199127137661, 0.054426535964012146, 0.05119955912232399, 0.045594945549964905, 0.04216785728931427, 0.03987661004066467, 0.038280684500932693, 0.037128377705812454, 0.0444006584584713, 0.04134046658873558, 0.03898823261260986, 0.037284720689058304, 0.04289891570806503, 0.04009697213768959, 0.0398666113615036, 0.04139731824398041, 0.10984703153371811, 0.08613982051610947, 0.06802891194820404, 0.05425377935171127, 0.06492927670478821, 0.06215813383460045, 0.05860862508416176, 0.055789485573768616, 0.0531456358730793, 0.035605430603027344, 0.03200441971421242, 0.029813406988978386 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 2.3435878802047228e-06, "first_to_last_ratio": 0.6055171886488473, "imbalance_abs_log_ratio": 0.501672328862846, "last_quartile_mean": 3.870390344218322e-06, "mean": 3.073230551819961e-06, "normalized": [ 0.8547974044010749, 0.7663300811036056, 0.7290646815405452, 0.7137467051168226, 0.7859991290227282, 0.7639513079235324, 0.7497289788008704, 0.7370314113250152, 0.6717110337337076, 0.6626923079350717, 0.6608463763960438, 0.6722646652249519, 0.7125587982314265, 0.7094665114737027, 0.7127329594676137, 0.7286490325070278, 0.7469367761442406, 0.7618697334205042, 0.7778837629071197, 0.8192912256851089, 2.1535525157084, 1.8967505152013653, 1.6654482647641247, 1.4715900882252853, 1.7756214494183695, 1.7029115748228523, 1.5831075845454274, 1.4223873934306603, 1.2671946390004933, 0.8569947657922223, 0.7658932722937289, 0.7009950544363569 ], "peak_layer": 21, "peak_normalized": 2.1535525157084, "population_cv": 0.43604978994955873, "rest_mean": 2.622152831261499e-06, "spike_contrast": 2.1009645880119394, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.509050242835656e-06, "top_five_layers": [ 21, 22, 25, 26, 23 ], "top_five_spike_overlap": 4 }, "values": [ 2.6269894988217857e-06, 2.355109018026269e-06, 2.240583853563294e-06, 2.1935081804258516e-06, 2.4155565370165277e-06, 2.3477984996134182e-06, 2.304090003235615e-06, 2.265067450935021e-06, 2.0643228708649985e-06, 2.036606247202144e-06, 2.0309332739998354e-06, 2.0660243080783403e-06, 2.189857468692935e-06, 2.1803541585541097e-06, 2.1903927063249284e-06, 2.2393064682546537e-06, 2.2955089207243873e-06, 2.341401341254823e-06, 2.390616145930835e-06, 2.5178708256134996e-06, 6.618363386223791e-06, 5.829151632497087e-06, 5.1183064897486474e-06, 4.522535618889378e-06, 5.456894086819375e-06, 5.233439878793433e-06, 4.865254595642909e-06, 4.371324394014664e-06, 3.8943812796787824e-06, 2.6337424969824497e-06, 2.353766603846452e-06, 2.1543194179685088e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 4.391086804389488e-05, "first_to_last_ratio": 1.2249727492111975, "imbalance_abs_log_ratio": 0.20291859820737496, "last_quartile_mean": 3.5846403989125974e-05, "mean": 4.0721109257901844e-05, "normalized": [ 1.0306443764454492, 1.1633240673546572, 1.0582852044477982, 0.9887262106756542, 1.2392117389406443, 1.1467748415480514, 1.060594525482963, 0.9390936315286891, 0.8770713560763073, 0.8238829996644348, 0.7830477106120073, 0.7532242441242194, 0.9307545606143657, 0.8564234405543825, 0.7975968172907832, 0.753337793856731, 0.905195060793232, 0.8405331784196935, 0.8351271926887619, 0.8717037860691994, 2.2508269312594313, 1.7299877984011494, 1.3168512568718882, 1.0054575984842986, 1.2551832994676113, 1.166557599233908, 1.0841041444130814, 1.0230520893344814, 0.8327362159057484, 0.6053498390287645, 0.5500890436554917, 0.5252514467561209 ], "peak_layer": 21, "peak_normalized": 2.2508269312594313, "population_cv": 0.3253138051630999, "rest_mean": 3.686269836314885e-05, "spike_contrast": 1.6698866556959722, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.155652808956802e-05, "top_five_layers": [ 21, 22, 23, 25, 5 ], "top_five_spike_overlap": 4 }, "values": [ 4.1968982259277254e-05, 4.7371846449095756e-05, 4.309454743633978e-05, 4.0262028051074594e-05, 5.046207661507651e-05, 4.669794361689128e-05, 4.3188585550524294e-05, 3.8240934372879565e-05, 3.571531851775944e-05, 3.3549429645063356e-05, 3.1886571377981454e-05, 3.067212674068287e-05, 3.790135815506801e-05, 3.487451249384321e-05, 3.247902714065276e-05, 3.0676750611746684e-05, 3.68605469702743e-05, 3.4227443393319845e-05, 3.400730565772392e-05, 3.5496745113050565e-05, 9.165616938844323e-05, 7.044702215353027e-05, 5.362364390748553e-05, 4.094334872206673e-05, 5.111245627631433e-05, 4.750351945403963e-05, 4.4145923311589286e-05, 4.165981590631418e-05, 3.390994243090972e-05, 2.4650516934343614e-05, 2.2400236048270017e-05, 2.138882155122701e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 3.186411663591571e-05, "first_to_last_ratio": 1.3918388457654427, "imbalance_abs_log_ratio": 0.3306257834899371, "last_quartile_mean": 2.2893538812240877e-05, "mean": 2.867843713261209e-05, "normalized": [ 1.079986899097868, 1.2037314747239194, 1.0983095858777547, 1.0266969289005365, 1.2873609620400022, 1.18616598037398, 1.0656281508421883, 0.9407819737650294, 0.8970835093851751, 0.8289921967977883, 0.7873997016130704, 0.7372847005450625, 0.9597295440949108, 0.8722370941591678, 0.8075308211758899, 0.7557215560612454, 0.9469694118556817, 0.867121573648186, 0.8716705000917596, 0.9227972264445797, 2.422954378421729, 1.8005437232549515, 1.3146220460386757, 0.9324076969576974, 1.239788118166858, 1.1164024129352315, 1.0295685897537663, 0.971835996891768, 0.628835627656002, 0.5006495076255713, 0.4607170575801389, 0.43847505322381475 ], "peak_layer": 21, "peak_normalized": 2.422954378421729, "population_cv": 0.3678541404959485, "rest_mean": 2.5799636171507666e-05, "spike_contrast": 1.7141312392387753, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.422396232257597e-05, "top_five_layers": [ 21, 22, 23, 5, 25 ], "top_five_spike_overlap": 4 }, "values": [ 3.0972336389822885e-05, 3.452113742241636e-05, 3.1497802410740405e-05, 2.944406332971994e-05, 3.691950041684322e-05, 3.401738649699837e-05, 3.056054993066937e-05, 2.698015669011511e-05, 2.572695302660577e-05, 2.3774200599291362e-05, 2.2581392840947956e-05, 2.1144172933418304e-05, 2.7523543394636363e-05, 2.5014396669575945e-05, 2.3158721887739375e-05, 2.1672913135262206e-05, 2.7157602744409814e-05, 2.4867691536201164e-05, 2.499814763723407e-05, 2.646438224473968e-05, 6.948654481675476e-05, 5.1636779971886426e-05, 3.770130570046604e-05, 2.673999551916495e-05, 3.5555185604607686e-05, 3.2016676414059475e-05, 2.9526418074965477e-05, 2.7870737540069968e-05, 1.8034023014479317e-05, 1.435784542991314e-05, 1.3212645171734039e-05, 1.2574779248097911e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00012265027817193186, "first_to_last_ratio": 1.047954536154132, "imbalance_abs_log_ratio": 0.04684020342552777, "last_quartile_mean": 0.00011703778545779642, "mean": 0.00011905099290743237, "normalized": [ 0.979179513665573, 1.0774208069918527, 1.0020137290002347, 0.935715487250064, 1.1754029671556891, 1.0964796843916984, 1.0415357532983844, 0.9341171734613325, 0.8608816802640402, 0.8236191260167243, 0.7913242278330239, 0.7785681534464967, 0.9070870788496911, 0.8497323537172862, 0.801684542793187, 0.7588960349730761, 0.8650613637183672, 0.8171925013555025, 0.8124116778573562, 0.8299123337690748, 2.0229186322404695, 1.6131445174416825, 1.308704488430075, 1.052279878697261, 1.3398531501841462, 1.2961845674727177, 1.2277152298587164, 1.1667193170961094, 0.9938188260868958, 0.6663087950093275, 0.6011322209314033, 0.5729841867425411 ], "peak_layer": 21, "peak_normalized": 2.0229186322404695, "population_cv": 0.2904848878678691, "rest_mean": 0.00010874690606552004, "spike_contrast": 1.6064186851302822, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.000174693061853759, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 0.00011657229333650321, 0.00012826801685150713, 0.0001192907293443568, 0.000111397857835982, 0.0001399328903062269, 0.00013053699512965977, 0.0001239958655787632, 0.00011120757699245587, 0.00010248881881125271, 9.805267472984269e-05, 9.420793503522873e-05, 9.268931171391159e-05, 0.00010798961739055812, 0.00010116148041561246, 9.544134081806988e-05, 9.034732647705823e-05, 0.0001029864142765291, 9.728757868288085e-05, 9.671841689851135e-05, 9.880188736133277e-05, 0.00024083047173917294, 0.00019204645650461316, 0.00015580256877001375, 0.0001252749643754214, 0.0001595108478795737, 0.00015431205974891782, 0.00014616071712225676, 0.00013889909314457327, 0.00011831511801574379, 7.932472362881526e-05, 7.156538777053356e-05, 6.821433635195717e-05 ] } } }, "post_attention_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "90a633f12d984935028fe5d7212be48e7da7b4a0806751939a551938d16dc540", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.0933690071105957, 0.35791492462158203, 0.3940892517566681, 0.4057151973247528, 0.09380833059549332, 0.11286329478025436, 0.13231655955314636, 0.1621972620487213, 0.05156813934445381, 0.08533531427383423, 0.12003090232610703, 0.16063299775123596, 0.046871740370988846, 0.079384446144104, 0.11459959298372269, 0.15625107288360596, 0.08397205173969269, 0.11654024571180344, 0.15294702351093292, 0.21572710573673248, 0.03708760067820549, 0.06553087383508682, 0.09390194714069366, 0.12852425873279572, 0.025487789884209633, 0.06059965491294861, 0.08860157430171967, 0.12100546807050705, 0.024166326969861984, 0.06877554953098297, 0.12682415544986725, 0.2025238573551178 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.4685734981867427e-05, "first_to_last_ratio": 0.9343190184571798, "imbalance_abs_log_ratio": 0.0679373375548271, "last_quartile_mean": 1.571811628764408e-05, "mean": 1.5263790061226246e-05, "normalized": [ 0.8950547219371277, 1.041381347295407, 0.928958474199763, 0.8623731199892722, 1.0959888418975217, 1.0227086828058576, 0.9773018451393299, 0.8732649216878685, 0.8344077395598781, 0.7634761278163229, 0.7254375260122781, 0.7001354215138343, 0.8361048428344902, 0.7831768738977604, 0.7468216112065258, 0.709083556727709, 0.7946157265465782, 0.7530502221399228, 0.7518984418471677, 0.7443354222404004, 2.2459882547048236, 1.9127017683069296, 1.5395014769185815, 1.2241132868423816, 1.2339903540150794, 1.1949497093148493, 1.1469092095200355, 1.0833954084554955, 1.3715915733231843, 0.9787904005078558, 0.6473265632240377, 0.5811665275717305 ], "peak_layer": 21, "peak_normalized": 2.2459882547048236, "population_cv": 0.35579152087029586, "rest_mean": 1.3479455750105748e-05, "spike_contrast": 1.8471958959531136, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.4899195341276937e-05, "top_five_layers": [ 21, 22, 23, 29, 25 ], "top_five_spike_overlap": 4 }, "values": [ 1.366192736895755e-05, 1.5895426258794032e-05, 1.417942712578224e-05, 1.3163082257960923e-05, 1.6728943592170253e-05, 1.5610410628141835e-05, 1.4917330190655775e-05, 1.3329332432476804e-05, 1.2736224562104326e-05, 1.1653539331746288e-05, 1.1072926099586766e-05, 1.0686720088415314e-05, 1.2762128790200222e-05, 1.1954247383982874e-05, 1.139932828664314e-05, 1.0823302545759361e-05, 1.2128847629355732e-05, 1.149440049630357e-05, 1.1476819963718299e-05, 1.1361379620211665e-05, 3.428229319979437e-05, 2.9195078241173178e-05, 2.3498627342632972e-05, 1.868460822151974e-05, 1.8835369701264426e-05, 1.823946149670519e-05, 1.7506181393400766e-05, 1.653672006796114e-05, 2.0935685824952088e-05, 1.4940051187295467e-05, 9.88065676210681e-06, 8.870803867466748e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 5.8224604345014086e-05, "first_to_last_ratio": 0.9289081115945783, "imbalance_abs_log_ratio": 0.07374545615334986, "last_quartile_mean": 6.268069319048664e-05, "mean": 6.0844937252113596e-05, "normalized": [ 0.8949455032066205, 1.0228957025766747, 0.9240146251144026, 0.8631043515939589, 1.0909185922403617, 1.0193622944527896, 0.9709926421810221, 0.8692402819766828, 0.8335854818875464, 0.7628111904092388, 0.7274012344088276, 0.7023925183778059, 0.8420599573247814, 0.7882546467013403, 0.7515041767890315, 0.712152858765499, 0.8000954144558701, 0.7563093989816041, 0.7544296301646483, 0.749005690846281, 2.243976282350735, 1.9218460060428357, 1.5396705122036691, 1.217662568253653, 1.226701769556092, 1.1864002531789521, 1.1411333345570556, 1.081567640735575, 1.3741638006355663, 0.9846717435990311, 0.6560255053997539, 0.5907043910320935 ], "peak_layer": 21, "peak_normalized": 2.243976282350735, "population_cv": 0.35448947038088385, "rest_mean": 5.3746683180280444e-05, "spike_contrast": 1.845239619854345, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.917550924001261e-05, "top_five_layers": [ 21, 22, 23, 29, 25 ], "top_five_spike_overlap": 4 }, "values": [ 5.445290298666805e-05, 6.223802483873442e-05, 5.622161188512109e-05, 5.251553011476062e-05, 6.63768732920289e-05, 6.202303484315053e-05, 5.907998638576828e-05, 5.2888870413880795e-05, 5.0719456339720637e-05, 4.641319901566021e-05, 4.425868246471509e-05, 4.273702870705165e-05, 5.123508526594378e-05, 4.796130451723002e-05, 4.5725224481429905e-05, 4.333089600550011e-05, 4.8681755288271233e-05, 4.6017597924219444e-05, 4.590322350850329e-05, 4.557320426101796e-05, 0.00013653459609486163, 0.00011693459964590147, 9.368115570396185e-05, 7.4088602559641e-05, 7.463859219569713e-05, 7.218644896056503e-05, 6.94321861374192e-05, 6.58079152344726e-05, 8.361091022379696e-05, 5.9912290453212336e-05, 3.991583071183413e-05, 3.59413716068957e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.0516341608017683, "first_to_last_ratio": 0.9289081033997755, "imbalance_abs_log_ratio": 0.0737454649753235, "last_quartile_mean": 0.05558586539700627, "mean": 0.05395789828617126, "normalized": [ 0.8949454811271385, 1.0228956842898203, 0.9240146476114748, 0.8631044001512534, 1.0909185722350807, 1.0193623200753072, 0.9709926227718745, 0.8692402530145229, 0.8335855026400435, 0.7628112052868975, 0.7274012041433464, 0.7023925275091049, 0.8420599706511929, 0.7882546311192083, 0.7515042050621248, 0.7121528782920565, 0.8000953848853934, 0.756309436789236, 0.7544295969980609, 0.7490056916860841, 2.2439762470957687, 1.9218459394336658, 1.539670533144279, 1.2176625655772975, 1.2267017863977538, 1.1864002526699235, 1.1411333128240784, 1.0815676327185337, 1.374163853918565, 0.984671791917794, 0.6560254933767684, 0.5907043745863505 ], "peak_layer": 21, "peak_normalized": 2.2439762470957687, "population_cv": 0.3544894636657234, "rest_mean": 0.04766309578661566, "spike_contrast": 1.8452395995744117, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.08794983178377151, "top_five_layers": [ 21, 22, 23, 29, 25 ], "top_five_spike_overlap": 4 }, "values": [ 0.048289377242326736, 0.055193301290273666, 0.049857888370752335, 0.04657129943370819, 0.058863673359155655, 0.05500264838337898, 0.05239272117614746, 0.0469023771584034, 0.04497852176427841, 0.04115968942642212, 0.03924904018640518, 0.037899624556303024, 0.04543578624725342, 0.04253256320953369, 0.040549587458372116, 0.038426272571086884, 0.043171465396881104, 0.040808867663145065, 0.040707435458898544, 0.04041477292776108, 0.12108024209737778, 0.10369876772165298, 0.08307738602161407, 0.06570251286029816, 0.06619025021791458, 0.06401566416025162, 0.06157315522432327, 0.058359116315841675, 0.07414699345827103, 0.05313082039356232, 0.03539775684475899, 0.03187316656112671 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 2.36006533782529e-06, "first_to_last_ratio": 0.5413470130102759, "imbalance_abs_log_ratio": 0.6136947769087335, "last_quartile_mean": 4.359616440297032e-06, "mean": 3.3537533212779636e-06, "normalized": [ 0.7861837298368325, 0.7027959957670996, 0.6663579384930622, 0.6515752574372438, 0.7209640412257049, 0.7064128183761409, 0.6971161849453695, 0.6982635098500108, 0.6495515913568419, 0.623944001859886, 0.618714632843181, 0.6237091538146412, 0.6696228280181353, 0.6705088641334079, 0.6781845443558197, 0.6899923705105712, 0.6887397572068925, 0.713528022619904, 0.7336582424796279, 0.7558738234893762, 2.2740550458877316, 2.1439073304121976, 1.8607357744257407, 1.5762316049354352, 1.650927113866359, 1.59960657867663, 1.5045046458050566, 1.3664264698086035, 1.623741147538761, 1.1718720847053645, 0.7902855706085994, 0.6920093247097722 ], "peak_layer": 21, "peak_normalized": 2.2740550458877316, "population_cv": 0.49135603522260235, "rest_mean": 2.794066934560154e-06, "spike_contrast": 2.281999665322214, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.376059809554136e-06, "top_five_layers": [ 21, 22, 23, 25, 29 ], "top_five_spike_overlap": 4 }, "values": [ 2.636666295074974e-06, 2.357004404984764e-06, 2.2348001493810443e-06, 2.1852226836927002e-06, 2.4179355477826903e-06, 2.3691343358223094e-06, 2.337955720577156e-06, 2.341803565286682e-06, 2.178435806854395e-06, 2.0925542685290566e-06, 2.0750162548210938e-06, 2.0917666461173212e-06, 2.245749783469364e-06, 2.2487213300337316e-06, 2.2744636680727126e-06, 2.314064204256283e-06, 2.309863248228794e-06, 2.392996975686401e-06, 2.4605087673990056e-06, 2.5350143459945684e-06, 7.626619662914891e-06, 7.19013632988208e-06, 6.240448783501051e-06, 5.286291980155511e-06, 5.536802291317144e-06, 5.364685875974828e-06, 5.045737452746835e-06, 4.582657311402727e-06, 5.445627266453812e-06, 3.930169896193547e-06, 2.6504228571866406e-06, 2.3208285711007193e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 4.4239922772248974e-05, "first_to_last_ratio": 1.1136909413511122, "imbalance_abs_log_ratio": 0.10767967155166762, "last_quartile_mean": 3.97236981370952e-05, "mean": 4.334811853823339e-05, "normalized": [ 0.9700332433430392, 1.0939339124231833, 0.9932423734988828, 0.930089943200089, 1.1681073372439403, 1.087889318229992, 1.0182311273894067, 0.9030573705493353, 0.8761983542241331, 0.798278529803464, 0.7546440525278227, 0.7221238940652536, 0.893949607919827, 0.8268303077758486, 0.7794623553075173, 0.7297655757222962, 0.8552687034485138, 0.8019698747855996, 0.7991515147639991, 0.7951190997107606, 2.3062412956458074, 1.9277971958458113, 1.5041468765724328, 1.133363677190868, 1.2016200733962084, 1.1258723047113217, 1.0645320609394298, 1.001245099310917, 1.090861764041589, 0.7755253626634432, 0.5600069277760075, 0.5114408659732598 ], "peak_layer": 21, "peak_normalized": 2.3062412956458074, "population_cv": 0.3513275217114342, "rest_mean": 3.841418893662868e-05, "spike_contrast": 1.8220178617427676, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.999133838689887e-05, "top_five_layers": [ 21, 22, 23, 25, 5 ], "top_five_spike_overlap": 4 }, "values": [ 4.2049116018461064e-05, 4.741997690871358e-05, 4.3055188143625855e-05, 4.031764910905622e-05, 5.06352553202305e-05, 4.7157955123111606e-05, 4.413840360939503e-05, 3.914583794539794e-05, 3.798155012191273e-05, 3.460387233644724e-05, 3.271239984314889e-05, 3.1302712159231305e-05, 3.875103357131593e-05, 3.584153819247149e-05, 3.3788226573960856e-05, 3.1633964681532234e-05, 3.707428913912736e-05, 3.476388519629836e-05, 3.464171459199861e-05, 3.4466916986275464e-05, 9.997122106142342e-05, 8.356638136319816e-05, 6.520193710457534e-05, 4.912918302579783e-05, 5.2087969379499555e-05, 4.88044461235404e-05, 4.61454619653523e-05, 4.340209125075489e-05, 4.728680505650118e-05, 3.3617565350141376e-05, 2.427524668746628e-05, 2.2169999283505604e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 3.203806136298226e-05, "first_to_last_ratio": 1.2891739083355929, "imbalance_abs_log_ratio": 0.25400163210680266, "last_quartile_mean": 2.4851620992194512e-05, "mean": 3.021096927113831e-05, "normalized": [ 1.026843472463997, 1.142743398382145, 1.0414039522523677, 0.9750373540158819, 1.2279493295544204, 1.1369004210816336, 1.0286142358538826, 0.9043300071962476, 0.9009657373841689, 0.8114808326453387, 0.7636098911314678, 0.7153533073356447, 0.9293664701105255, 0.8536797920313359, 0.7979693239477054, 0.7370852885386994, 0.9029339881565371, 0.8397847483866244, 0.8415951898817099, 0.8395201273360108, 2.469141151457409, 1.9893975767816492, 1.4975200883202109, 1.0459537484550225, 1.18622578491601, 1.089289946158196, 1.015117659129071, 0.9537627848996331, 0.8411713747337533, 0.5950625986834817, 0.464034356542757, 0.43615606223646275 ], "peak_layer": 21, "peak_normalized": 2.469141151457409, "population_cv": 0.38466525104607585, "rest_mean": 2.664357035192316e-05, "spike_contrast": 1.8569179273426082, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.947492343490012e-05, "top_five_layers": [ 21, 22, 23, 5, 25 ], "top_five_spike_overlap": 4 }, "values": [ 3.1021936592878774e-05, 3.452338569331914e-05, 3.146182280033827e-05, 2.9456823540385813e-05, 3.709753946168348e-05, 3.434686368564144e-05, 3.107543307123706e-05, 2.7320686058374122e-05, 2.7219048206461594e-05, 2.4515622499166057e-05, 2.3069394956110045e-05, 2.1611516785924323e-05, 2.8077061870135367e-05, 2.5790493964450434e-05, 2.4107426725095138e-05, 2.226806100225076e-05, 2.7278510970063508e-05, 2.5370711227878928e-05, 2.542540642025415e-05, 2.5362716769450344e-05, 7.459514745278284e-05, 6.0101629060227424e-05, 4.524153337115422e-05, 3.1599276553606614e-05, 3.58370307367295e-05, 3.2908505090745166e-05, 3.066768840653822e-05, 2.8814098186558113e-05, 2.541260255384259e-05, 1.7977417883230373e-05, 1.401892768626567e-05, 1.3176697393646464e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00012353622514638118, "first_to_last_ratio": 0.9287361711371687, "imbalance_abs_log_ratio": 0.07393057281911733, "last_quartile_mean": 0.0001330154127572314, "mean": 0.00012799764363080612, "normalized": [ 0.9123056916284747, 1.0004206383850174, 0.9261902365047731, 0.8739289130473129, 1.0942783540947973, 1.0293888052890365, 0.9950959036297305, 0.8895476688715391, 0.8565026169148985, 0.7932981654390305, 0.7514405160241083, 0.7360674988234528, 0.8681894979175296, 0.8094703021331294, 0.7698466402052128, 0.727570501836618, 0.8142391548620697, 0.7716594102042104, 0.7740031643537453, 0.7568672865946415, 2.079370086584246, 1.810928693076996, 1.4595869229947036, 1.1861869867777464, 1.2798737143358492, 1.2476425614336344, 1.1967044634398516, 1.1414050306889945, 1.3512106760869314, 0.9238435264430416, 0.6139160757906545, 0.5590202955880221 ], "peak_layer": 21, "peak_normalized": 2.079370086584246, "population_cv": 0.3241823234463846, "rest_mean": 0.00011464821754661561, "spike_contrast": 1.7452041450542493, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00020008454448543488, "top_five_layers": [ 21, 22, 23, 29, 25 ], "top_five_spike_overlap": 4 }, "values": [ 0.00011677297879941761, 0.00012805148435290903, 0.00011855016782646999, 0.00011186084157088771, 0.00014006505080033094, 0.00013175934145692736, 0.00012736993085127324, 0.0001138600055128336, 0.00010963031672872603, 0.0001015402958728373, 9.618261537980288e-05, 9.421490540262312e-05, 0.00011112620995845646, 0.00010361029126215726, 9.853855590336025e-05, 9.312730981037021e-05, 0.00010422069317428395, 9.877058619167656e-05, 9.907058120006695e-05, 9.687722922535613e-05, 0.0002661544713191688, 0.00023179460549727082, 0.00018682368681766093, 0.00015182913921307772, 0.00016382081957999617, 0.00015969530795700848, 0.00015317535144276917, 0.00014609715435653925, 0.00017295178258791566, 0.00011824979446828365, 7.857981108827516e-05, 7.155328057706356e-05 ] } } }, "post_mlp_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "89f2226456bfe96e245d2aad0f699d32c676f9f53099789cd870253d002402f5", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.16915453970432281, 0.374446302652359, 0.39966297149658203, 0.4194087088108063, 0.09892481565475464, 0.11664740741252899, 0.13951337337493896, 0.18593810498714447, 0.06607726216316223, 0.10425819456577301, 0.14551182091236115, 0.19865906238555908, 0.06484340131282806, 0.10430748760700226, 0.14720691740512848, 0.20439592003822327, 0.10052042454481125, 0.14085541665554047, 0.1745898574590683, 0.2420409768819809, 0.049456026405096054, 0.08509506285190582, 0.11783231794834137, 0.15649692714214325, 0.05533855780959129, 0.08507220447063446, 0.11942370235919952, 0.1559232622385025, 0.06628894805908203, 0.12384890764951706, 0.19928614795207977, 0.30097496509552 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.4557030681316974e-05, "first_to_last_ratio": 1.0521214637846872, "imbalance_abs_log_ratio": 0.050808567521915836, "last_quartile_mean": 1.3835884146828903e-05, "mean": 1.4168486075050168e-05, "normalized": [ 0.9626776420770625, 1.1208331393048514, 1.0015907146010516, 0.9273210288804931, 1.175791874817754, 1.0895388135257325, 1.0279799843086574, 0.9136520518095224, 0.8429430620475944, 0.7964245945861879, 0.7616346629097698, 0.7379957371546335, 0.8789024323209899, 0.8184444211135518, 0.7730235620926829, 0.7408880724440507, 0.8509573560882475, 0.7987944762176407, 0.7947809742504978, 0.821305623582707, 2.191634231664042, 1.7068550093167172, 1.3663058625551863, 1.0875228189831554, 1.3033590230854797, 1.249186503619112, 1.1759131965297032, 1.1148864496778903, 1.0543371583252714, 0.7016042304977244, 0.6285050746276369, 0.5844102169844034 ], "peak_layer": 21, "peak_normalized": 2.191634231664042, "population_cv": 0.317485077550744, "rest_mean": 1.2774896377878678e-05, "spike_contrast": 1.69816410232038, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.169387043977622e-05, "top_five_layers": [ 21, 22, 23, 25, 26 ], "top_five_spike_overlap": 4 }, "values": [ 1.363968476653099e-05, 1.5880508726695552e-05, 1.4191024092724547e-05, 1.3138735084794462e-05, 1.665919080551248e-05, 1.543711550766602e-05, 1.4564920093107503e-05, 1.2945066373504233e-05, 1.1943227036681492e-05, 1.128413077822188e-05, 1.0791210115712602e-05, 1.0456282325321808e-05, 1.2452716873667669e-05, 1.1596118383749854e-05, 1.0952573575195856e-05, 1.0497262337594293e-05, 1.2056777450197842e-05, 1.1317708413116634e-05, 1.1260843166382983e-05, 1.1636657291091979e-05, 3.105213909293525e-05, 2.4183551431633532e-05, 1.9358485587872565e-05, 1.540855191706214e-05, 1.846662416937761e-05, 1.7699081581667997e-05, 1.666090975049883e-05, 1.579625313752331e-05, 1.4938361346139573e-05, 9.940669770003296e-06, 8.90496539796004e-06, 8.280208021460567e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 5.772678514404106e-05, "first_to_last_ratio": 1.044603064835594, "imbalance_abs_log_ratio": 0.04363697099583948, "last_quartile_mean": 5.526193353944109e-05, "mean": 5.6508551210754376e-05, "normalized": [ 0.9614821271309634, 1.1002365609935876, 0.9957674960909164, 0.927558955897375, 1.169788310103388, 1.0860790463100811, 1.021700170878454, 0.9098545301693725, 0.8414652130089573, 0.7957499499255085, 0.763899533401845, 0.7409106647351306, 0.886022872587649, 0.8249615248910556, 0.7780050940211537, 0.7440194105244476, 0.856059090328315, 0.8001537489433276, 0.7955484428363649, 0.8260861662138186, 2.1920257903000504, 1.7189392921930309, 1.357513601934732, 1.0826579323429608, 1.2956647027296744, 1.2403754064999633, 1.1695452140111815, 1.1132942203700882, 1.0605287199578224, 0.7105276447235593, 0.638647233338548, 0.5949313326066779 ], "peak_layer": 21, "peak_normalized": 2.1920257903000504, "population_cv": 0.31542805831527876, "rest_mean": 5.09690361030624e-05, "spike_contrast": 1.6955771464373153, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 8.642193279229105e-05, "top_five_layers": [ 21, 22, 23, 25, 26 ], "top_five_spike_overlap": 4 }, "values": [ 5.433196201920509e-05, 6.217277405085042e-05, 5.6269378546858206e-05, 5.241501276032068e-05, 6.610304262721911e-05, 6.13727534073405e-05, 5.7734796428121626e-05, 5.141456131241284e-05, 4.7549980081385e-05, 4.4966676796320826e-05, 4.3166855903109536e-05, 4.186778824077919e-05, 5.006786886951886e-05, 4.6617380576208234e-05, 4.396394069772214e-05, 4.2043458961416036e-05, 4.8374658945249394e-05, 4.521552909864113e-05, 4.495528992265463e-05, 4.668093242798932e-05, 0.00012386820162646472, 9.713476902106777e-05, 7.671112689422444e-05, 6.117943121353164e-05, 7.321613520616665e-05, 7.009181717876345e-05, 6.608930561924353e-05, 6.291064346441999e-05, 5.9928941482212394e-05, 4.015088779851794e-05, 3.608902989071794e-05, 3.36187076754868e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.05119269015267491, "first_to_last_ratio": 1.0446030503106722, "imbalance_abs_log_ratio": 0.043636957091111164, "last_quartile_mean": 0.04900683579035103, "mean": 0.050112348108086735, "normalized": [ 0.9614821831138475, 1.1002365372531873, 0.9957675209931669, 0.9275589493810882, 1.1697883360925696, 1.0860790159442548, 1.0217001547239526, 0.9098545023598117, 0.8414652131989159, 0.7957499921436116, 0.7638995463226855, 0.7409106535279784, 0.8860229052732944, 0.824961560322453, 0.7780051048316907, 0.7440194265324953, 0.8560591030852058, 0.8001537465139836, 0.7955484597393159, 0.8260861572284607, 2.1920256627654124, 1.7189392343107897, 1.3575135033616252, 1.0826579457687873, 1.295664687824762, 1.24037545097571, 1.1695453226548556, 1.1132942164663777, 1.0605287065053515, 0.7105276550183023, 0.6386471942222375, 0.594931351543819 ], "peak_layer": 21, "peak_normalized": 2.1920256627654124, "population_cv": 0.3154280355761813, "rest_mean": 0.04519985126400435, "spike_contrast": 1.6955770632627056, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.07663983106613159, "top_five_layers": [ 21, 22, 23, 25, 26 ], "top_five_spike_overlap": 4 }, "values": [ 0.048182129859924316, 0.05513543635606766, 0.049900248646736145, 0.046482156962156296, 0.058620840311050415, 0.05442596971988678, 0.05119979381561279, 0.045594945549964905, 0.042167797684669495, 0.03987690061330795, 0.038280799984931946, 0.03712877258658409, 0.044400688260793686, 0.04134076088666916, 0.0389876626431942, 0.03728456050157547, 0.042899131774902344, 0.04009758308529854, 0.03986680135130882, 0.04139711707830429, 0.1098475530743599, 0.08614008128643036, 0.06802818924188614, 0.05425453186035156, 0.06492879986763, 0.06215812638401985, 0.05860866233706474, 0.05578978732228279, 0.05314558371901512, 0.03560620918869972, 0.032004110515117645, 0.029813406988978386 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 2.3435262619386776e-06, "first_to_last_ratio": 0.6055057813720351, "imbalance_abs_log_ratio": 0.5016911679387263, "last_quartile_mean": 3.87036149618325e-06, "mean": 3.0732030609215144e-06, "normalized": [ 0.8547503013075527, 0.7663054921948581, 0.7290555922607795, 0.7137661113532094, 0.7859807829039316, 0.7639110127263136, 0.7497406424491107, 0.737033935109932, 0.6716719850193362, 0.6626792955655075, 0.6608508821708053, 0.672311075170408, 0.7125289932221522, 0.7094835118736261, 0.7127088529301003, 0.7286376459464239, 0.7469440496474942, 0.7618938613164015, 0.7778685995728054, 0.8192523873398297, 2.1536522766569326, 1.896755940512382, 1.6655003037120943, 1.4715957055627773, 1.775576812524924, 1.702936426140992, 1.583063297151765, 1.422456494444769, 1.2671802274178405, 0.857037723187697, 0.7658684575198893, 0.7010013250873588 ], "peak_layer": 21, "peak_normalized": 2.1536522766569326, "population_cv": 0.43606223483790696, "rest_mean": 2.622115920934852e-06, "spike_contrast": 2.1010030765098153, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.509073616849491e-06, "top_five_layers": [ 21, 22, 25, 26, 23 ], "top_five_spike_overlap": 4 }, "values": [ 2.6268212423019577e-06, 2.3550123842142057e-06, 2.2405358777177753e-06, 2.1935481981927296e-06, 2.4154785478458507e-06, 2.347653662582161e-06, 2.3041052372718696e-06, 2.2650549453828717e-06, 2.0641844002966536e-06, 2.0365480395412305e-06, 2.030928953900002e-06, 2.0661484541051323e-06, 2.1897462829656433e-06, 2.1803869003633736e-06, 2.190299028370646e-06, 2.2392514438251965e-06, 2.295510739713791e-06, 2.3414545466948766e-06, 2.3905481612018775e-06, 2.517728944440023e-06, 6.618610768782673e-06, 5.829116162203718e-06, 5.11842063133372e-06, 4.522512426774483e-06, 5.456708095152862e-06, 5.233469437371241e-06, 4.865074970439309e-06, 4.371497652755352e-06, 3.894302153639728e-06, 2.633850954225636e-06, 2.3536692879133625e-06, 2.1543194179685088e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 4.391093034428195e-05, "first_to_last_ratio": 1.2249725680030026, "imbalance_abs_log_ratio": 0.20291845027901623, "last_quartile_mean": 3.584646015042381e-05, "mean": 4.07211941251262e-05, "normalized": [ 1.0306449979747543, 1.1633207494786566, 1.0582852323384906, 0.988727812953862, 1.2392193409064014, 1.1467684313076425, 1.0605911536899448, 0.939091138327177, 0.8770695281679378, 0.8238853028479478, 0.7830494735295855, 0.7532269625840828, 0.9307494046311563, 0.8564271053395186, 0.7975879185804384, 0.7533353304349726, 0.9051971945139116, 0.8405383057410994, 0.83513125921233, 0.871701790669994, 2.2508301021070363, 1.7299888385385092, 1.316850031174253, 1.0054625607633896, 1.2551720176799122, 1.1665564187478432, 1.084102510397689, 1.0230582656867047, 0.832740912783307, 0.6053538930694665, 0.550085663744774, 0.5252503520772114 ], "peak_layer": 21, "peak_normalized": 2.2508301021070363, "population_cv": 0.32531388254580146, "rest_mean": 3.686278021767201e-05, "spike_contrast": 1.6698856912552844, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.155662922537885e-05, "top_five_layers": [ 21, 22, 23, 25, 5 ], "top_five_spike_overlap": 4 }, "values": [ 4.1969095036620274e-05, 4.7371810069307685e-05, 4.309463838580996e-05, 4.0262177208205685e-05, 5.046249134466052e-05, 4.669777990784496e-05, 4.31885382567998e-05, 3.824091254500672e-05, 3.571531851775944e-05, 3.3549593354109675e-05, 3.1886709621176124e-05, 3.067230136366561e-05, 3.7901227187830955e-05, 3.4874734410550445e-05, 3.2478732464369386e-05, 3.067671423195861e-05, 3.686071067932062e-05, 3.422772351768799e-05, 3.400754212634638e-05, 3.549673783709295e-05, 9.165648953057826e-05, 7.044721132842824e-05, 5.362370575312525e-05, 4.094363612239249e-05, 5.1112103392370045e-05, 4.750357038574293e-05, 4.4145948777440935e-05, 4.166015423834324e-05, 3.3910204365383834e-05, 2.4650733394082636e-05, 2.240014509879984e-05, 2.138882155122701e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 3.186627918694285e-05, "first_to_last_ratio": 1.3918711207627403, "imbalance_abs_log_ratio": 0.3306489719669603, "last_quartile_mean": 2.2894561652719858e-05, "mean": 2.8678776743618073e-05, "normalized": [ 1.0800662050319052, 1.2041245441213098, 1.0980135715526718, 1.0269427257275898, 1.2874205603122744, 1.1862286797761374, 1.065650543118825, 0.9407131151701387, 0.8970071765459219, 0.8291042219304858, 0.7873071619721381, 0.7371817181801269, 0.9595979227688407, 0.8721850306992133, 0.8077112837508718, 0.7556571722797087, 0.9468894438093267, 0.8670643698070588, 0.8714483973650358, 0.9225828905554825, 2.423160363456308, 1.8005876036705257, 1.314694260458331, 0.9321789763393963, 1.2397768617501317, 1.1166059837948594, 1.0296033331928969, 0.971833621904611, 0.628858435408621, 0.5007221007775481, 0.46061186392256265, 0.43846986084914497 ], "peak_layer": 21, "peak_normalized": 2.423160363456308, "population_cv": 0.36790402733408, "rest_mean": 2.579985448392108e-05, "spike_contrast": 1.714155286168129, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.422495694598183e-05, "top_five_layers": [ 21, 22, 23, 5, 25 ], "top_five_spike_overlap": 4 }, "values": [ 3.0974977562436834e-05, 3.453281897236593e-05, 3.1489686080021784e-05, 2.945146115962416e-05, 3.6921646824339405e-05, 3.401958747417666e-05, 3.0561554012820125e-05, 2.6978501409757882e-05, 2.5725068553583696e-05, 2.3777694877935573e-05, 2.2579006326850504e-05, 2.1141469915164635e-05, 2.752009459072724e-05, 2.501319977454841e-05, 2.31641715799924e-05, 2.1671323338523507e-05, 2.715563095989637e-05, 2.486634548404254e-05, 2.499207403161563e-05, 2.645854874572251e-05, 6.949327507754788e-05, 5.163864989299327e-05, 3.7703823181800544e-05, 2.673375274753198e-05, 3.555528383003548e-05, 3.2022893719840795e-05, 2.95277641271241e-05, 2.7870999474544078e-05, 1.8034890672424808e-05, 1.436009733879473e-05, 1.3209784810896963e-05, 1.2574779248097911e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.0001226487638632534, "first_to_last_ratio": 1.047953014836365, "imbalance_abs_log_ratio": 0.04683875172240625, "last_quartile_mean": 0.00011703651034622453, "mean": 0.00011905245355592342, "normalized": [ 0.9791591884455346, 1.077487405094885, 1.0019863398009818, 0.9356419747088983, 1.1755798379057922, 1.0961957342605828, 1.041360407354167, 0.9342513512096913, 0.8606870380776002, 0.8236454459505925, 0.7914262383638166, 0.7785606791625734, 0.9069383787142993, 0.8496268325797314, 0.8013864859756553, 0.7587096112174009, 0.8651412624719578, 0.8173040341152568, 0.8123620464200654, 0.8299529386489826, 2.023626833206166, 1.6131874304204101, 1.3089111371095015, 1.0523372512253804, 1.339898805005123, 1.2960324991779766, 1.2277957518221394, 1.166763795846918, 0.9937141651289164, 0.66627672391588, 0.6010752198343956, 0.5729771568287279 ], "peak_layer": 21, "peak_normalized": 2.023626833206166, "population_cv": 0.29058780865825207, "rest_mean": 0.00010874356289773627, "spike_contrast": 1.606720052702735, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00017472046311013402, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 0.00011657130380626768, 0.00012827751925215125, 0.0001192889321828261, 0.0001113904727390036, 0.0001399556640535593, 0.0001305047917412594, 0.00012397651153150946, 0.00011122491559945047, 0.00010246690362691879, 9.805701120058075e-05, 9.422123548574746e-05, 9.268955909647048e-05, 0.0001079732392099686, 0.00010115015902556479, 9.540702740196139e-05, 9.032624075189233e-05, 0.00010299718996975571, 9.730205056257546e-05, 9.671369480201975e-05, 9.880793368211016e-05, 0.00024091773957479745, 0.0001920539216371253, 0.00015582908235955983, 0.00012528333172667772, 0.0001595182402525097, 0.00015429584891535342, 0.00014617209671996534, 0.00013890609261579812, 0.00011830410949187353, 7.932187872938812e-05, 7.155947969295084e-05, 6.821433635195717e-05 ] } } }, "pre_attention_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "ee695e78937de01b7128d3866b8fc5dd328ebdf2c159be37918d4f02952d5248", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.0626930519938469, 0.09030644595623016, 0.20229491591453552, 0.20300470292568207, 0.19049881398677826, 0.12695962190628052, 0.12132706493139267, 0.11585728824138641, 0.13427142798900604, 0.10485932976007462, 0.10419609397649765, 0.10852954536676407, 0.1286265105009079, 0.09096142649650574, 0.10080574452877045, 0.10566592961549759, 0.1287592649459839, 0.09650235623121262, 0.1065765842795372, 0.11446042358875275, 0.12395420670509338, 0.07221997529268265, 0.07677769660949707, 0.08416415005922318, 0.09094838052988052, 0.059439923614263535, 0.062182098627090454, 0.06896103918552399, 0.0776243805885315, 0.06285123527050018, 0.07231374084949493, 0.08016443997621536 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 9.208858216425142e-06, "first_to_last_ratio": 1.5742443936121653, "imbalance_abs_log_ratio": 0.45377540707318936, "last_quartile_mean": 5.84970049999356e-06, "mean": 9.688216671577266e-06, "normalized": [ 0.8835618522393698, 1.1875758123727602, 0.8027516569014074, 0.7168128072164557, 0.4410707043730502, 1.0021885510288027, 1.1732636073567753, 1.396946986573399, 0.8742295075093847, 0.7201495961759767, 0.6464112962338885, 0.5634673377355092, 0.5113744900448981, 0.9719461816017829, 0.7757782507499562, 0.6793505932199849, 0.8733188127550255, 1.2655600470684725, 1.3182773604333435, 1.4727865215105207, 1.811417959722297, 3.2379187324814804, 2.107963115128669, 1.7355151521056127, 0.6684050298520982, 0.8511096328466135, 0.6952697740946564, 0.6802290414380979, 0.6372906896771284, 0.38940384422203217, 0.4352345037562327, 0.4734205515743182 ], "peak_layer": 22, "peak_normalized": 3.2379187324814804, "population_cv": 0.5794583443248681, "rest_mean": 8.051546762578289e-06, "spike_contrast": 2.300953434969459, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.852623418017174e-05, "top_five_layers": [ 22, 23, 21, 24, 20 ], "top_five_spike_overlap": 4 }, "values": [ 8.560138667235151e-06, 1.150549178419169e-05, 7.777231985528488e-06, 6.944637789274566e-06, 4.273188551451312e-06, 9.709419828141108e-06, 1.1366832040948793e-05, 1.3533925084630027e-05, 8.469724889437202e-06, 6.976965323701734e-06, 6.262572696869029e-06, 5.458993655338418e-06, 4.954306859872304e-06, 9.416425200470258e-06, 7.515907782362774e-06, 6.581695743079763e-06, 8.460901881335303e-06, 1.2261019946890883e-05, 1.277175670111319e-05, 1.4268674931372516e-05, 1.7549409676576033e-05, 3.136965824523941e-05, 2.042240339505952e-05, 1.681404683040455e-05, 6.4756527535791975e-06, 8.245734534284566e-06, 6.735924216627609e-06, 6.590206339751603e-06, 6.174210284370929e-06, 3.7726288155681686e-06, 4.216646175336791e-06, 4.586600880429614e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 3.5088153708784375e-05, "first_to_last_ratio": 1.5310723408421485, "imbalance_abs_log_ratio": 0.4259683662726232, "last_quartile_mean": 2.2917371552466648e-05, "mean": 3.811287126609386e-05, "normalized": [ 0.7220272140535721, 1.1083207213140864, 0.7679551692742715, 0.7110701915728868, 0.44898497189039965, 0.967564647567629, 1.2046976563462615, 1.4344826160998425, 0.8528241284869815, 0.7193089103145273, 0.6221437263787414, 0.5634827545988164, 0.521538663554135, 0.9942025542377431, 0.7750755634633926, 0.7193187896753318, 0.8819670974271809, 1.3528979955012577, 1.312160761445291, 1.5169165753693319, 1.905325315822389, 3.1582493344310123, 2.1798331396840407, 1.7492300787723019, 0.6501223625354283, 0.8905765546940848, 0.6694712099870339, 0.6213878836876245, 0.640037682841144, 0.40616882441760144, 0.44459600619501544, 0.48806089836064387 ], "peak_layer": 22, "peak_normalized": 3.1582493344310123, "population_cv": 0.5843705435205574, "rest_mean": 3.1559207450805436e-05, "spike_contrast": 2.329039979322277, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 7.350265586865134e-05, "top_five_layers": [ 22, 23, 21, 24, 20 ], "top_five_spike_overlap": 4 }, "values": [ 2.751853025984019e-05, 4.224128497298807e-05, 2.9268976504681632e-05, 2.7100926672574133e-05, 1.7112106434069574e-05, 3.687666685436852e-05, 4.5914486690890044e-05, 5.467225128086284e-05, 3.2503576221643016e-05, 2.7414927899371833e-05, 2.3711683752480894e-05, 2.1475945686688647e-05, 1.987733594432939e-05, 3.78919139620848e-05, 2.9540355171775445e-05, 2.7415304430178367e-05, 3.361429844517261e-05, 5.1562827138695866e-05, 5.001021418138407e-05, 5.781404615845531e-05, 7.261741848196834e-05, 0.00012036995030939579, 8.307969983434305e-05, 6.666818080702797e-05, 2.4778029910521582e-05, 3.394242958165705e-05, 2.5515470042591915e-05, 2.368287641729694e-05, 2.4393673811573535e-05, 1.5480260117328726e-05, 1.6944830349530093e-05, 1.860140218923334e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.031116526224650443, "first_to_last_ratio": 1.5310723794021894, "imbalance_abs_log_ratio": 0.42596839145761245, "last_quartile_mean": 0.02032335416879505, "mean": 0.03379887552000582, "normalized": [ 0.7220271626647913, 1.108320765743466, 0.7679551826054016, 0.7110702013600861, 0.4489849675608935, 0.9675646960926859, 1.2046976729559553, 1.4344826560026684, 0.8528240868421063, 0.7193088777937033, 0.6221436817460179, 0.5634827360150323, 0.5215386498121982, 0.994202510087396, 0.775075573451505, 0.719318797535757, 0.8819670765827853, 1.352897957921212, 1.3121607736682377, 1.5169165942232727, 1.905325416965852, 3.158249458986568, 2.17983311618398, 1.7492300093001025, 0.6501223693179898, 0.8905764758208182, 0.6694712119613627, 0.6213879076208866, 0.6400376839978682, 0.4061688258121883, 0.4445960052440992, 0.4880608981231125 ], "peak_layer": 22, "peak_normalized": 3.158249458986568, "population_cv": 0.5843705625933613, "rest_mean": 0.027987020711104076, "spike_contrast": 2.329040027551536, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.06518289148807525, "top_five_layers": [ 22, 23, 21, 24, 20 ], "top_five_spike_overlap": 4 }, "values": [ 0.024403706192970276, 0.03745999559760094, 0.02595602162182331, 0.024033373221755028, 0.015175187028944492, 0.032702598720788956, 0.04071742668747902, 0.04848390072584152, 0.028824495151638985, 0.02431183122098446, 0.021027756854891777, 0.019045082852244377, 0.017627419903874397, 0.03360292688012123, 0.026196682825684547, 0.02431216649711132, 0.029809495434165, 0.04572642967104912, 0.044349558651447296, 0.051270075142383575, 0.06439785659313202, 0.10674528032541275, 0.07367590814828873, 0.059122007340192795, 0.02197340503334999, 0.03010048344731331, 0.022627374157309532, 0.021002212539315224, 0.02163255400955677, 0.013728049583733082, 0.015026845037937164, 0.01649590954184532 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 2.778968109851121e-06, "first_to_last_ratio": 0.6551322428275108, "imbalance_abs_log_ratio": 0.4229181662788503, "last_quartile_mean": 4.241842987084965e-06, "mean": 3.2108540644060213e-06, "normalized": [ 3.044896087768068, 1.305426400048544, 0.24778795376225207, 0.34289699866506496, 0.26296263207516013, 0.4634963504907376, 0.5461366646920124, 0.7103319256958529, 0.492164757900996, 0.5211240695504333, 0.4284048228004182, 0.539518062145093, 0.39350580318561185, 0.6925294077014227, 0.6296611634917264, 0.6921288832669542, 0.45745559083371623, 0.8998038475442661, 0.6921375225846708, 0.8582476671161742, 1.1275308806840705, 2.206285306565431, 2.0329194098054084, 1.8438884472309407, 1.0018897080158793, 1.4436105873878202, 1.6464990155797314, 1.8044066236378706, 1.722118113784269, 1.2644804245447474, 0.8237033552263064, 0.8620515162183501 ], "peak_layer": 1, "peak_normalized": 3.044896087768068, "population_cv": 0.6539975553973914, "rest_mean": 2.8288202555712714e-06, "spike_contrast": 1.864323695267318, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.273836632113671e-06, "top_five_layers": [ 1, 22, 23, 24, 28 ], "top_five_spike_overlap": 3 }, "values": [ 9.776716979104094e-06, 4.191533662378788e-06, 7.956109584483784e-07, 1.10099222183635e-06, 8.443346359854331e-07, 1.4882191408105427e-06, 1.7535651295474963e-06, 2.280772150697885e-06, 1.5802692132638185e-06, 1.6732533367758151e-06, 1.375545366499864e-06, 1.7323137626590324e-06, 1.2634897075258777e-06, 2.2236108634388074e-06, 2.021750105996034e-06, 2.2223248379305005e-06, 1.4688231431136956e-06, 2.889138841055683e-06, 2.2223525775189046e-06, 2.7557080102269538e-06, 3.620337110987748e-06, 7.0840601438248996e-06, 6.527407549583586e-06, 5.920456715102773e-06, 3.216921641069348e-06, 4.635222921933746e-06, 5.286668056214694e-06, 5.793686341348803e-06, 5.529469945031451e-06, 4.060062110511353e-06, 2.6447912659932626e-06, 2.7679216145770624e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 2.3473811893381935e-05, "first_to_last_ratio": 1.6221250197474661, "imbalance_abs_log_ratio": 0.48373703024908266, "last_quartile_mean": 1.447102511065168e-05, "mean": 2.567504009221011e-05, "normalized": [ 0.6038503331656763, 1.1237726617309638, 0.7965709499671938, 0.7286242389585822, 0.5100708602864626, 1.0419074268588182, 1.2699253805041595, 1.2394048051027409, 0.9067624283684929, 0.7783780379317236, 0.6557328600555824, 0.5841298974967862, 0.5749474691248395, 1.0479010495751726, 0.8238301727903069, 0.7175888176534647, 0.8799730602245153, 1.337737482964768, 1.313729694195371, 1.482381735841077, 1.8384597441691042, 3.1947987115358343, 2.299711378461891, 1.740832415098614, 0.6549479505275152, 0.8864147168659078, 0.6253692098824938, 0.5590121549791961, 0.6490066126050694, 0.40394742932233174, 0.33873950633935634, 0.39154080741598873 ], "peak_layer": 22, "peak_normalized": 3.1947987115358343, "population_cv": 0.5935289341881325, "rest_mean": 2.1178341908294157e-05, "spike_contrast": 2.3588820362651397, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.995721028535627e-05, "top_five_layers": [ 22, 23, 21, 24, 20 ], "top_five_spike_overlap": 4 }, "values": [ 1.5503881513723172e-05, 2.8852908144472167e-05, 2.0451991076697595e-05, 1.8707456547417678e-05, 1.3096089787723031e-05, 2.675101495697163e-05, 3.260538505855948e-05, 3.182176806149073e-05, 2.3281161702470854e-05, 1.9984887330792844e-05, 1.683596747170668e-05, 1.4997558537288569e-05, 1.476179932069499e-05, 2.690490146051161e-05, 2.1151872715563513e-05, 1.8424121662974358e-05, 2.2593343601329252e-05, 3.434646350797266e-05, 3.373006256879307e-05, 3.806021049967967e-05, 4.720252763945609e-05, 8.202658500522375e-05, 5.904518184252083e-05, 4.469594205147587e-05, 1.6815814888104796e-05, 2.2758733393857256e-05, 1.6056379536166787e-05, 1.4352659491123632e-05, 1.6663270798744634e-05, 1.0371366442996077e-05, 8.697150406078435e-06, 1.0052825928141829e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 1.5368138974736212e-05, "first_to_last_ratio": 1.6684179576216203, "imbalance_abs_log_ratio": 0.5118758466617863, "last_quartile_mean": 9.21120448538204e-06, "mean": 1.70536165029489e-05, "normalized": [ 0.5060443334956218, 1.129033308485111, 0.8092732940368782, 0.7332465948386053, 0.5799019345143905, 1.1120711228491045, 1.314528534872588, 1.0252286917222369, 0.9618738734749688, 0.8573870936806717, 0.6935167713776637, 0.6090324452150211, 0.6280013879885763, 1.0922315969057461, 0.8788466701599956, 0.716368246297853, 0.8747898507186472, 1.3340749509714696, 1.3260980540176734, 1.3930719468730541, 1.7547500822965236, 3.2343903364608173, 2.4337162111509074, 1.681466434725036, 0.647867793000642, 0.880473920446508, 0.5990376981746858, 0.5105448708965856, 0.6551326612911179, 0.4134969533643083, 0.2892689108887433, 0.3252334248082473 ], "peak_layer": 22, "peak_normalized": 3.2343903364608173, "population_cv": 0.602068868215782, "rest_mean": 1.4052059449804128e-05, "spike_contrast": 2.3670569220652067, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.326202458993066e-05, "top_five_layers": [ 22, 23, 21, 24, 20 ], "top_five_spike_overlap": 4 }, "values": [ 8.629885996924713e-06, 1.9254101061960682e-05, 1.3801036402583122e-05, 1.2504506230470724e-05, 9.889425200526603e-06, 1.89648344530724e-05, 2.2417465515900403e-05, 1.7483856936451048e-05, 1.6403428162448108e-05, 1.4621550690208096e-05, 1.1826969057437964e-05, 1.0386205758550204e-05, 1.07096948340768e-05, 1.8626498786034063e-05, 1.4987514077802189e-05, 1.2216669347253628e-05, 1.4918330634827726e-05, 2.2750802600057796e-05, 2.2614767658524215e-05, 2.3756914742989466e-05, 2.9924834962002933e-05, 5.515805241884664e-05, 4.150366294197738e-05, 2.8675083740381524e-05, 1.104848888644483e-05, 1.5015264580142684e-05, 1.0215759175480343e-05, 8.706636435817927e-06, 1.117238116421504e-05, 7.051618467812659e-06, 4.933081072522327e-06, 5.546406100620516e-06 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 7.076486235746415e-05, "first_to_last_ratio": 1.6194330115644955, "imbalance_abs_log_ratio": 0.4820760951210741, "last_quartile_mean": 4.36973075466085e-05, "mean": 7.626050199860401e-05, "normalized": [ 0.7582408131347899, 1.1294507430341896, 0.7848163930039677, 0.7314607712340419, 0.4478687849408395, 1.0030187883056183, 1.211588283674721, 1.3570431263074092, 0.866372986696188, 0.6928870960227855, 0.6156993397811549, 0.5449854830682977, 0.5247931162318564, 0.9825868965367193, 0.7730573932527033, 0.719766458957068, 0.8392396443274492, 1.325527923124894, 1.310979443011046, 1.574679744732153, 1.925673026139939, 3.2689210238039985, 2.1805315614636265, 1.8468069734137842, 0.6742654040322257, 0.8994888709350041, 0.6396994899493578, 0.5797806655888059, 0.6397902241500218, 0.366359490386347, 0.3506259557794401, 0.4339940849795572 ], "peak_layer": 22, "peak_normalized": 3.2689210238039985, "population_cv": 0.6088945491433702, "rest_mean": 6.243137175732516e-05, "spike_contrast": 2.41765960050045, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001509378053015098, "top_five_layers": [ 22, 23, 21, 24, 20 ], "top_five_spike_overlap": 4 }, "values": [ 5.7823825045488775e-05, 8.61324806464836e-05, 5.985049210721627e-05, 5.5781565606594086e-05, 3.415469836909324e-05, 7.649071631021798e-05, 9.239633072866127e-05, 0.00010348879004595801, 6.607003888348117e-05, 5.283991777105257e-05, 4.695354073191993e-05, 4.156086652074009e-05, 4.002098648925312e-05, 7.49325699871406e-05, 5.895374488318339e-05, 5.488975148182362e-05, 6.400083657354116e-05, 0.0001010854248306714, 9.997595043387264e-05, 0.00012008586782030761, 0.00014685279165860265, 0.0002492895582690835, 0.00016628843150101602, 0.00014083842688705772, 5.141981819178909e-05, 6.859547283966094e-05, 4.878380423178896e-05, 4.42143646068871e-05, 4.879072366748005e-05, 2.7938758648815565e-05, 2.6738911401480436e-05, 3.309660678496584e-05 ] } } }, "pre_mlp_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "e53fa425d89dfeb0e9395eca9bcd73dc0a55314abf45c5b10b43e2cbfa724345", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.059514161199331284, 0.08157797157764435, 0.14292451739311218, 0.10997853428125381, 0.0991024449467659, 0.1065012738108635, 0.12155740708112717, 0.1095810979604721, 0.06964563578367233, 0.08311934769153595, 0.09363401681184769, 0.0941167026758194, 0.08564268052577972, 0.09493856877088547, 0.09063401818275452, 0.09473779052495956, 0.10505300015211105, 0.09355910867452621, 0.09618847072124481, 0.11416846513748169, 0.047205954790115356, 0.053352247923612595, 0.06512638926506042, 0.07829347252845764, 0.05368548259139061, 0.05310077592730522, 0.061235878616571426, 0.07320034503936768, 0.05304349213838577, 0.0639304369688034, 0.07404901832342148, 0.08157160878181458 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 4.264712657686687e-06, "first_to_last_ratio": 0.1644294405719314, "imbalance_abs_log_ratio": 1.8052737334880782, "last_quartile_mean": 2.593642989268119e-05, "mean": 1.2728848709286922e-05, "normalized": [ 0.7308709553274441, 0.25839772175666137, 0.17673121478863804, 0.3160700993142839, 0.2515262179120628, 0.19919508634940433, 0.26852523246873117, 0.47902810031308257, 0.5562516846458733, 0.4204955008816364, 0.38437114805684625, 0.3903867171146432, 0.4547984133336864, 0.4706996478069255, 0.5281782311682919, 0.5336633446585687, 0.4098263345872015, 0.510432013069417, 0.47387948742830627, 0.4589311652228786, 1.8475604507418064, 2.103099989443774, 1.8619737802126157, 1.6142271653963092, 1.9601447862493215, 2.46815888223834, 2.9205894659583342, 2.1524913761211453, 2.4429733892073924, 1.9340088460256004, 1.2584531027710544, 1.1640604494297233 ], "peak_layer": 27, "peak_normalized": 2.9205894659583342, "population_cv": 0.8183880615733347, "rest_mean": 1.0660643603707151e-05, "spike_contrast": 2.2416241615194457, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.3897156279417685e-05, "top_five_layers": [ 27, 26, 29, 28, 22 ], "top_five_spike_overlap": 1 }, "values": [ 9.303145816375036e-06, 3.28910550706496e-06, 2.249584895253065e-06, 4.023208475700812e-06, 3.201639174221782e-06, 2.535524117774912e-06, 3.4180170587205794e-06, 6.097476216382347e-06, 7.0804435381432995e-06, 5.352423613658175e-06, 4.89260219183052e-06, 4.969173460267484e-06, 5.789060196548235e-06, 5.991464604448993e-06, 6.723100796079962e-06, 6.792919975850964e-06, 5.21661741004209e-06, 6.497211870737374e-06, 6.031940301909344e-06, 5.841665370098781e-06, 2.3517317458754405e-05, 2.677004158613272e-05, 2.3700782548985444e-05, 2.0547253370750695e-05, 2.4950386432465166e-05, 3.1416821002494544e-05, 3.7175741454120725e-05, 2.739873707469087e-05, 3.1096238672034815e-05, 2.4617706003482454e-05, 1.6018659152905457e-05, 1.4817149349255487e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 1.7121947848863783e-05, "first_to_last_ratio": 0.16509620138165101, "imbalance_abs_log_ratio": 1.8012269363058968, "last_quartile_mean": 0.00010370891459388076, "mean": 5.084153968937244e-05, "normalized": [ 0.7383170312690549, 0.26576827119147, 0.1760185135778732, 0.31888712811432673, 0.24735537154739307, 0.19775481444509202, 0.2688888848514836, 0.4811766768028211, 0.5612046705561601, 0.420641548111133, 0.38487619754759494, 0.39104050297970294, 0.44984735784914776, 0.46682630828073346, 0.5246515683050852, 0.5270843392878012, 0.4096339186170339, 0.5134146758073647, 0.47618140517897295, 0.458935947238964, 1.8467859080778408, 2.0931308659053878, 1.8587343459276247, 1.6040752411223071, 1.9556157106583185, 2.466644949277254, 2.929922797201843, 2.162838410955913, 2.4514855413823686, 1.9278621503865954, 1.264951753177563, 1.1594471943677749 ], "peak_layer": 27, "peak_normalized": 2.929922797201843, "population_cv": 0.8187130280816768, "rest_mean": 4.263469445167748e-05, "spike_contrast": 2.2319499458540437, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.515850397292524e-05, "top_five_layers": [ 27, 26, 29, 28, 22 ], "top_five_spike_overlap": 1 }, "values": [ 3.753717464860529e-05, 1.351206810795702e-05, 8.949052244133782e-06, 1.6212712580454536e-05, 1.257592793990625e-05, 1.0054159247374628e-05, 1.3670724911207799e-05, 2.4463763111270964e-05, 2.8532509531942196e-05, 2.1386063963291235e-05, 1.9567698473110795e-05, 1.988110125239473e-05, 2.287093229824677e-05, 2.3734168280498125e-05, 2.6674093533074483e-05, 2.679777935554739e-05, 2.082641913148109e-05, 2.6102792617166415e-05, 2.420979581074789e-05, 2.3333010176429525e-05, 9.389343904331326e-05, 0.00010641799599397928, 9.450091602047905e-05, 8.155365503625944e-05, 9.942651377059519e-05, 0.0001254080270882696, 0.00014896178618073463, 0.00010996203491231427, 0.00012463729945011437, 9.801548003451899e-05, 6.431209476431832e-05, 5.894808055018075e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.015183915151283145, "first_to_last_ratio": 0.1650962046167148, "imbalance_abs_log_ratio": 1.8012269167108745, "last_quartile_mean": 0.09197010425850749, "mean": 0.04508678655838594, "normalized": [ 0.7383170831605016, 0.26576827430432287, 0.17601850704343852, 0.3188871482222363, 0.24735536125956567, 0.1977548199529969, 0.268888871402825, 0.48117668466109587, 0.5612046640272622, 0.4206415668248819, 0.38487618974904236, 0.3910405014958363, 0.4498473547312449, 0.4668263129953462, 0.524651582664394, 0.5270843482987495, 0.40963390858947524, 0.5134146403888652, 0.4761813910194076, 0.4589359262070489, 1.84678585895072, 2.093130779904821, 1.8587344079994206, 1.6040752759385333, 1.9556157145006774, 2.466644974623479, 2.929922838238967, 2.162838250844473, 2.4514856227642747, 1.9278621827857403, 1.2649517868659736, 1.1594471695843824 ], "peak_layer": 27, "peak_normalized": 2.929922838238967, "population_cv": 0.8187130295488606, "rest_mean": 0.03780887407009249, "spike_contrast": 2.2319499342595517, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0843875139951706, "top_five_layers": [ 27, 26, 29, 28, 22 ], "top_five_spike_overlap": 1 }, "values": [ 0.033288344740867615, 0.011982637457549572, 0.007936108857393265, 0.014377596788108349, 0.011152458377182484, 0.008916129358112812, 0.012123335152864456, 0.021694710478186607, 0.025302914902567863, 0.018965376541018486, 0.01735283061861992, 0.01763075962662697, 0.020282171666622162, 0.021047698333859444, 0.02365485392510891, 0.023764539510011673, 0.018469076603651047, 0.02314821630716324, 0.021469488739967346, 0.020691946148872375, 0.08326563984155655, 0.09437254071235657, 0.08380436152219772, 0.07232259958982468, 0.08817242830991745, 0.11121309548616409, 0.13210080564022064, 0.09751542657613754, 0.11052960902452469, 0.08692111074924469, 0.05703261122107506, 0.05227574706077576 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 1.0203952811593808e-06, "first_to_last_ratio": 0.14653458073549716, "imbalance_abs_log_ratio": 1.9204938324033933, "last_quartile_mean": 6.963511793856014e-06, "mean": 3.3145427504877034e-06, "normalized": [ 0.7458840109402727, 0.2408134163028247, 0.17355607700807005, 0.34010416858400555, 0.23617982419644817, 0.18519537363627844, 0.21382216991189298, 0.32727704932881246, 0.45837194023673944, 0.38199948313264875, 0.3580283506292727, 0.3663182076515519, 0.42179968724485917, 0.4793449276202126, 0.5258832837607723, 0.5351040608812003, 0.41254056339182255, 0.5133680832769055, 0.4649030953676889, 0.3708867509400463, 1.824270688162059, 2.097875379911376, 1.8960504208985058, 1.623249707156654, 1.9999231446844747, 2.472735888191646, 3.1676028298394496, 2.259810270376047, 2.461216776671721, 1.9178482403264976, 1.3254841043068921, 1.2025520254323514 ], "peak_layer": 27, "peak_normalized": 3.1676028298394496, "population_cv": 0.8575358529508568, "rest_mean": 2.7693165078643587e-06, "spike_contrast": 2.2600394150975536, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.258764460653765e-06, "top_five_layers": [ 27, 26, 29, 28, 22 ], "top_five_spike_overlap": 1 }, "values": [ 2.4722644411667716e-06, 7.98186363226705e-07, 5.752590368501842e-07, 1.1272898063907633e-06, 7.828281241017976e-07, 6.138379831099883e-07, 7.087227231750148e-07, 1.084773771253822e-06, 1.5192933915386675e-06, 1.2661536175073707e-06, 1.1867002740473254e-06, 1.2141773595431005e-06, 1.3980730955154286e-06, 1.5888092548266286e-06, 1.7430626257919357e-06, 1.7736252857503132e-06, 1.3673833336724783e-06, 1.7015804587572347e-06, 1.5409411844302667e-06, 1.2293199915802688e-06, 6.0466231843747664e-06, 6.953497631911887e-06, 6.284540177148301e-06, 5.380330549087375e-06, 6.628830760746496e-06, 8.195988812076394e-06, 1.0499154996068683e-05, 7.490237749152584e-06, 8.157808224495966e-06, 6.356789981509792e-06, 4.393373728817096e-06, 3.985910097981105e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 1.1562887948457501e-05, "first_to_last_ratio": 0.18471338835857254, "imbalance_abs_log_ratio": 1.6889499073516332, "last_quartile_mean": 6.259907877392834e-05, "mean": 3.280785603010372e-05, "normalized": [ 0.7954597876813794, 0.28319080316573975, 0.14214063507139948, 0.2974318261790621, 0.27466777359580663, 0.21803323898158788, 0.2807192887038541, 0.5278979023125309, 0.6226995573358239, 0.46211210539922204, 0.42141148445543547, 0.43822745218006914, 0.5122302293591944, 0.517641757670608, 0.5752168949015549, 0.5726437521109653, 0.45443775276735304, 0.575490842294968, 0.5223238130301624, 0.5079099996805576, 1.9644506441760023, 2.2223204692543925, 1.9528462747282453, 1.5940837481500663, 2.0842410031055634, 2.45962178512242, 2.7403579287154862, 2.0172743105168234, 2.1184828194144836, 1.705641602573747, 1.1156418609587828, 1.0231506564067125 ], "peak_layer": 27, "peak_normalized": 2.7403579287154862, "population_cv": 0.777507293302437, "rest_mean": 2.6953546694130637e-05, "spike_contrast": 2.3900797611316444, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.442112644435837e-05, "top_five_layers": [ 27, 26, 22, 29, 25 ], "top_five_spike_overlap": 2 }, "values": [ 2.6097330191987567e-05, 9.290883099311031e-06, 4.663329491449986e-06, 9.758100532053504e-06, 9.011260772240348e-06, 7.153203114285134e-06, 9.209798008669168e-06, 1.731919837766327e-05, 2.042943742708303e-05, 1.5160907423705794e-05, 1.382560731144622e-05, 1.4377303159562871e-05, 1.6805175619083457e-05, 1.6982716260827146e-05, 1.8871633074013516e-05, 1.8787213775794953e-05, 1.4909128367435187e-05, 1.8880620700656436e-05, 1.7136324458988383e-05, 1.666343814576976e-05, 6.44494139123708e-05, 7.290957000805065e-05, 6.406869943020865e-05, 5.2298470109235495e-05, 6.83794787619263e-05, 8.069491741480306e-05, 8.99052683962509e-05, 6.61824451526627e-05, 6.95028793415986e-05, 5.5958444136194885e-05, 3.6601817555492744e-05, 3.3567379432497546e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 7.540980021758514e-06, "first_to_last_ratio": 0.21360723680914365, "imbalance_abs_log_ratio": 1.543616292277398, "last_quartile_mean": 3.5303017511978396e-05, "mean": 2.0393022815312634e-05, "normalized": [ 0.8359603193367185, 0.295087194870454, 0.12666628430801963, 0.2806700274686079, 0.3079335868931964, 0.2413250265274421, 0.29661301513585586, 0.574003393126437, 0.6785255042280676, 0.49280857080169316, 0.44265641404459144, 0.4732900683545354, 0.5730266005666174, 0.5692791815349577, 0.6227366987056507, 0.6204634327774647, 0.5079971554671316, 0.6525690994711822, 0.5813750506103471, 0.5701698101268489, 2.1864401210834683, 2.4727028679652125, 2.1461264444780124, 1.6025169004567823, 2.1852134886953176, 2.4240833773167205, 2.4037042609517996, 1.7540846718423961, 1.7738254969500735, 1.4662718655044793, 0.9512460136887227, 0.890628056711195 ], "peak_layer": 22, "peak_normalized": 2.4727028679652125, "population_cv": 0.748243893154277, "rest_mean": 1.6168646037907888e-05, "spike_contrast": 2.6721258732489797, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.320465741329827e-05, "top_five_layers": [ 22, 26, 27, 21, 25 ], "top_five_spike_overlap": 3 }, "values": [ 1.7047757864929736e-05, 6.017719897499774e-06, 2.583108425824321e-06, 5.7237102737417445e-06, 6.2796966631140094e-06, 4.921346771880053e-06, 6.04883598498418e-06, 1.1705664292094298e-05, 1.383718608849449e-05, 1.004985642794054e-05, 9.027102350955829e-06, 9.651815162214916e-06, 1.1685744539136067e-05, 1.1609323337324895e-05, 1.2699483704636805e-05, 1.2653124940698035e-05, 1.0359597581555136e-05, 1.330785653408384e-05, 1.1855994671350345e-05, 1.16274859465193e-05, 4.458812327357009e-05, 5.0425886001903564e-05, 4.376600554678589e-05, 3.268016371293925e-05, 4.456310853129253e-05, 4.943438761983998e-05, 4.9018795834854245e-05, 3.577108873287216e-05, 3.617366382968612e-05, 2.9901715606683865e-05, 1.9398781660129316e-05, 1.8162598280468956e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 3.495021690014255e-05, "first_to_last_ratio": 0.1566784937110037, "imbalance_abs_log_ratio": 1.8535593840231388, "last_quartile_mean": 0.0002230696509286645, "mean": 0.00010673854976062103, "normalized": [ 0.738885306995076, 0.25891564745697115, 0.13953919908256351, 0.29899111537939826, 0.24198859123609487, 0.19717850597867767, 0.2681440510653662, 0.4758585581160271, 0.5790269925422016, 0.43589999516688577, 0.3997169856917177, 0.4056240269847417, 0.4457491198754207, 0.47489649494374236, 0.531638590252971, 0.5311180734156474, 0.42812373537534565, 0.5225858167691654, 0.4854761905276067, 0.45573352033875775, 1.692186877570602, 1.9495472798858815, 1.771674897459069, 1.552543580233036, 2.023811044874862, 2.6451452315044404, 3.2041309624863805, 2.3680255028880883, 2.341068783870195, 1.8377340464745107, 1.1954728471343063, 1.1035684284242502 ], "peak_layer": 27, "peak_normalized": 3.2041309624863805, "population_cv": 0.8392868954770956, "rest_mean": 9.096589831272097e-05, "spike_contrast": 2.1097012302295255, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00019191086757928134, "top_five_layers": [ 27, 26, 28, 29, 25 ], "top_five_spike_overlap": 1 }, "values": [ 7.886754610808566e-05, 2.7636280719889328e-05, 1.489421174483141e-05, 3.1913878046907485e-05, 2.5829511287156492e-05, 2.1046547772129998e-05, 2.8621307137655094e-05, 5.079245238448493e-05, 6.180450145620853e-05, 4.65273333247751e-05, 4.266521136742085e-05, 4.329572038841434e-05, 4.757861461257562e-05, 5.0689763156697154e-05, 5.674633212038316e-05, 5.6690772908041254e-05, 4.569730663206428e-05, 5.5780052207410336e-05, 5.1819024520227686e-05, 4.864433503826149e-05, 0.0001806215732358396, 0.00020809184934478253, 0.00018910600920207798, 0.00016571625019423664, 0.0002160186559194699, 0.00028233896591700613, 0.00034200429217889905, 0.00025275960797443986, 0.00024988228688016534, 0.000196157066966407, 0.00012760303798131645, 0.00011779329361161217 ] } } } }, "selector": { "expected_uniform_count": 4, "identities_unique": true, "identity_order_sha256": "97fd9721a263024f6a746638f32e0ad9a988cd4c111c00e9e178c1022e5c9d5e", "passed": true, "uniform_count": 4, "uniform_indices": [ 49, 51, 53, 55 ], "visit_count": 65, "visits": [ { "branch": "attention", "coefficient": "detached_learned", "group": 1, "index": 0, "kind": "depth", "layer": 1, "offset": 1, "sources": 1, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 1, "index": 1, "kind": "depth", "layer": 1, "offset": 1, "sources": 2, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 1, "index": 2, "kind": "depth", "layer": 2, "offset": 2, "sources": 2, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 1, "index": 3, "kind": "depth", "layer": 2, "offset": 2, "sources": 2, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 1, "index": 4, "kind": "depth", "layer": 3, "offset": 3, "sources": 2, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 1, "index": 5, "kind": "depth", "layer": 3, "offset": 3, "sources": 2, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 1, "index": 6, "kind": "depth", "layer": 4, "offset": 4, "sources": 2, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 1, "index": 7, "kind": "depth", "layer": 4, "offset": 4, "sources": 2, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 2, "index": 8, "kind": "depth", "layer": 5, "offset": 1, "sources": 2, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 2, "index": 9, "kind": "depth", "layer": 5, "offset": 1, "sources": 3, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 2, "index": 10, "kind": "depth", "layer": 6, "offset": 2, "sources": 3, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 2, "index": 11, "kind": "depth", "layer": 6, "offset": 2, "sources": 3, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 2, "index": 12, "kind": "depth", "layer": 7, "offset": 3, "sources": 3, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 2, "index": 13, "kind": "depth", "layer": 7, "offset": 3, "sources": 3, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 2, "index": 14, "kind": "depth", "layer": 8, "offset": 4, "sources": 3, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 2, "index": 15, "kind": "depth", "layer": 8, "offset": 4, "sources": 3, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 3, "index": 16, "kind": "depth", "layer": 9, "offset": 1, "sources": 3, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 3, "index": 17, "kind": "depth", "layer": 9, "offset": 1, "sources": 4, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 3, "index": 18, "kind": "depth", "layer": 10, "offset": 2, "sources": 4, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 3, "index": 19, "kind": "depth", "layer": 10, "offset": 2, "sources": 4, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 3, "index": 20, "kind": "depth", "layer": 11, "offset": 3, "sources": 4, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 3, "index": 21, "kind": "depth", "layer": 11, "offset": 3, "sources": 4, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 3, "index": 22, "kind": "depth", "layer": 12, "offset": 4, "sources": 4, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 3, "index": 23, "kind": "depth", "layer": 12, "offset": 4, "sources": 4, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 4, "index": 24, "kind": "depth", "layer": 13, "offset": 1, "sources": 4, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 4, "index": 25, "kind": "depth", "layer": 13, "offset": 1, "sources": 5, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 4, "index": 26, "kind": "depth", "layer": 14, "offset": 2, "sources": 5, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 4, "index": 27, "kind": "depth", "layer": 14, "offset": 2, "sources": 5, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 4, "index": 28, "kind": "depth", "layer": 15, "offset": 3, "sources": 5, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 4, "index": 29, "kind": "depth", "layer": 15, "offset": 3, "sources": 5, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 4, "index": 30, "kind": "depth", "layer": 16, "offset": 4, "sources": 5, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 4, "index": 31, "kind": "depth", "layer": 16, "offset": 4, "sources": 5, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 5, "index": 32, "kind": "depth", "layer": 17, "offset": 1, "sources": 5, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 5, "index": 33, "kind": "depth", "layer": 17, "offset": 1, "sources": 6, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 5, "index": 34, "kind": "depth", "layer": 18, "offset": 2, "sources": 6, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 5, "index": 35, "kind": "depth", "layer": 18, "offset": 2, "sources": 6, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 5, "index": 36, "kind": "depth", "layer": 19, "offset": 3, "sources": 6, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 5, "index": 37, "kind": "depth", "layer": 19, "offset": 3, "sources": 6, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 5, "index": 38, "kind": "depth", "layer": 20, "offset": 4, "sources": 6, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 5, "index": 39, "kind": "depth", "layer": 20, "offset": 4, "sources": 6, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 6, "index": 40, "kind": "depth", "layer": 21, "offset": 1, "sources": 6, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 6, "index": 41, "kind": "depth", "layer": 21, "offset": 1, "sources": 7, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 6, "index": 42, "kind": "depth", "layer": 22, "offset": 2, "sources": 7, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 6, "index": 43, "kind": "depth", "layer": 22, "offset": 2, "sources": 7, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 6, "index": 44, "kind": "depth", "layer": 23, "offset": 3, "sources": 7, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 6, "index": 45, "kind": "depth", "layer": 23, "offset": 3, "sources": 7, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 6, "index": 46, "kind": "depth", "layer": 24, "offset": 4, "sources": 7, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 6, "index": 47, "kind": "depth", "layer": 24, "offset": 4, "sources": 7, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 7, "index": 48, "kind": "depth", "layer": 25, "offset": 1, "sources": 7, "use_uniform": false }, { "branch": "mlp", "coefficient": "uniform", "group": 7, "index": 49, "kind": "depth", "layer": 25, "offset": 1, "sources": 8, "use_uniform": true }, { "branch": "attention", "coefficient": "detached_learned", "group": 7, "index": 50, "kind": "depth", "layer": 26, "offset": 2, "sources": 8, "use_uniform": false }, { "branch": "mlp", "coefficient": "uniform", "group": 7, "index": 51, "kind": "depth", "layer": 26, "offset": 2, "sources": 8, "use_uniform": true }, { "branch": "attention", "coefficient": "detached_learned", "group": 7, "index": 52, "kind": "depth", "layer": 27, "offset": 3, "sources": 8, "use_uniform": false }, { "branch": "mlp", "coefficient": "uniform", "group": 7, "index": 53, "kind": "depth", "layer": 27, "offset": 3, "sources": 8, "use_uniform": true }, { "branch": "attention", "coefficient": "detached_learned", "group": 7, "index": 54, "kind": "depth", "layer": 28, "offset": 4, "sources": 8, "use_uniform": false }, { "branch": "mlp", "coefficient": "uniform", "group": 7, "index": 55, "kind": "depth", "layer": 28, "offset": 4, "sources": 8, "use_uniform": true }, { "branch": "attention", "coefficient": "detached_learned", "group": 8, "index": 56, "kind": "depth", "layer": 29, "offset": 1, "sources": 8, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 8, "index": 57, "kind": "depth", "layer": 29, "offset": 1, "sources": 9, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 8, "index": 58, "kind": "depth", "layer": 30, "offset": 2, "sources": 9, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 8, "index": 59, "kind": "depth", "layer": 30, "offset": 2, "sources": 9, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 8, "index": 60, "kind": "depth", "layer": 31, "offset": 3, "sources": 9, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 8, "index": 61, "kind": "depth", "layer": 31, "offset": 3, "sources": 9, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 8, "index": 62, "kind": "depth", "layer": 32, "offset": 4, "sources": 9, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 8, "index": 63, "kind": "depth", "layer": 32, "offset": 4, "sources": 9, "use_uniform": false }, { "branch": "output", "coefficient": "detached_learned", "group": null, "index": 64, "kind": "output", "layer": null, "offset": null, "sources": 9, "use_uniform": false } ], "visits_sha256": "86ffbfde38ca8bed5f23de529d214f0cf29ea110bc34c6d2b71e42982072b897" } }, "uniform_group_7_only": { "bits_per_byte": 1.8740129107275643, "forward": { "activation_sha256": { "attention_branch_output": "5e6ecbcd78543fc42d4098a51169dca446d404ce384015643e84d2c78433c949", "mlp_branch_output": "c0b8018c349b1bd09e5d17858eb312a2323da0e92de9651e2bb08ae4adf115c7", "post_attention_state": "90a633f12d984935028fe5d7212be48e7da7b4a0806751939a551938d16dc540", "post_mlp_state": "89f2226456bfe96e245d2aad0f699d32c676f9f53099789cd870253d002402f5", "pre_attention_input": "ee695e78937de01b7128d3866b8fc5dd328ebdf2c159be37918d4f02952d5248", "pre_mlp_input": "e53fa425d89dfeb0e9395eca9bcd73dc0a55314abf45c5b10b43e2cbfa724345" }, "logits_sha256": "bdacba5675f0579cec629bbf01b36c8660ad576cfdf9fef1bea8259eaeac9ec2", "loss_nats": 1.2989667654037476, "mixer_summary_sha256": "eb3ac0f07de9b755d6e67a510ab68d09d9ad666bb7b5d020ba7d8c5055c399b2" }, "loss_nats": 1.2989667654037476, "loss_scale": 1.0, "mixers": [ { "branch": "attention", "entropy_mean": -0.0, "group": 1, "latest_source_mass": 1.0, "layer": 1, "max_source_mass": 1.0, "mixer_index": 0, "normalized_entropy": 1.0, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 1.0, "median": 1.0, "p05": 1.0, "p95": 1.0 } ], "sources": 1 }, { "branch": "mlp", "entropy_mean": 0.22886337339878082, "group": 1, "latest_source_mass": 0.06350618600845337, "layer": 1, "max_source_mass": 0.9364938139915466, "mixer_index": 1, "normalized_entropy": 0.3301800538435402, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.9364938139915466, "median": 0.9452826976776123, "p05": 0.8725647330284119, "p95": 0.9712384939193726 }, { "label": "current_group_partial", "mean": 0.06350618600845337, "median": 0.0547172985970974, "p05": 0.028761589899659157, "p95": 0.12743529677391052 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.603729784488678, "group": 1, "latest_source_mass": 0.4133155345916748, "layer": 2, "max_source_mass": 0.5866844654083252, "mixer_index": 2, "normalized_entropy": 0.8709979661187783, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.5866844654083252, "median": 0.5734395384788513, "p05": 0.29813364148139954, "p95": 0.8843642473220825 }, { "label": "current_group_partial", "mean": 0.4133155345916748, "median": 0.4265604615211487, "p05": 0.11563576757907867, "p95": 0.7018663287162781 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.3967982828617096, "group": 1, "latest_source_mass": 0.14110636711120605, "layer": 2, "max_source_mass": 0.858893632888794, "mixer_index": 3, "normalized_entropy": 0.5724589149178446, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.858893632888794, "median": 0.8705973029136658, "p05": 0.7599387168884277, "p95": 0.9270997643470764 }, { "label": "current_group_partial", "mean": 0.14110636711120605, "median": 0.12940272688865662, "p05": 0.07290027290582657, "p95": 0.2400612235069275 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6852496862411499, "group": 1, "latest_source_mass": 0.518975019454956, "layer": 3, "max_source_mass": 0.518975019454956, "mixer_index": 4, "normalized_entropy": 0.9886063241108252, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.48102498054504395, "median": 0.48102718591690063, "p05": 0.38786420226097107, "p95": 0.580143928527832 }, { "label": "current_group_partial", "mean": 0.518975019454956, "median": 0.5189728140830994, "p05": 0.41985607147216797, "p95": 0.6121357679367065 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.5890734195709229, "group": 1, "latest_source_mass": 0.3123423159122467, "layer": 3, "max_source_mass": 0.6876576542854309, "mixer_index": 5, "normalized_entropy": 0.8498533011344741, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.6876576542854309, "median": 0.6909762620925903, "p05": 0.48803940415382385, "p95": 0.8900076746940613 }, { "label": "current_group_partial", "mean": 0.3123423159122467, "median": 0.3090237081050873, "p05": 0.10999235510826111, "p95": 0.5119606256484985 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6873323321342468, "group": 1, "latest_source_mass": 0.4831485152244568, "layer": 4, "max_source_mass": 0.5168514251708984, "mixer_index": 6, "normalized_entropy": 0.9916109470127238, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.5168514251708984, "median": 0.5219584703445435, "p05": 0.4251725673675537, "p95": 0.5947866439819336 }, { "label": "current_group_partial", "mean": 0.4831485152244568, "median": 0.47804147005081177, "p05": 0.405213326215744, "p95": 0.5748274326324463 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.487301766872406, "group": 1, "latest_source_mass": 0.2060384750366211, "layer": 4, "max_source_mass": 0.7939615249633789, "mixer_index": 7, "normalized_entropy": 0.7030278424832499, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.7939615249633789, "median": 0.7953404188156128, "p05": 0.6590759754180908, "p95": 0.9329437017440796 }, { "label": "current_group_partial", "mean": 0.2060384750366211, "median": 0.20465964078903198, "p05": 0.06705628335475922, "p95": 0.3409239649772644 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.674697756767273, "group": 2, "latest_source_mass": 0.42952626943588257, "layer": 5, "max_source_mass": 0.5704737305641174, "mixer_index": 8, "normalized_entropy": 0.9733831077870527, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.5704737305641174, "median": 0.5696132779121399, "p05": 0.4695202708244324, "p95": 0.6750621199607849 }, { "label": "completed_group_1", "mean": 0.42952626943588257, "median": 0.4303867816925049, "p05": 0.3249378800392151, "p95": 0.5304797887802124 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.9691112041473389, "group": 2, "latest_source_mass": 0.2681158185005188, "layer": 5, "max_source_mass": 0.5506307482719421, "mixer_index": 9, "normalized_entropy": 0.8821230329784768, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.5506307482719421, "median": 0.543993353843689, "p05": 0.41262108087539673, "p95": 0.7161091566085815 }, { "label": "completed_group_1", "mean": 0.18125343322753906, "median": 0.1794438362121582, "p05": 0.11575750261545181, "p95": 0.25031501054763794 }, { "label": "current_group_partial", "mean": 0.2681158185005188, "median": 0.2664111256599426, "p05": 0.1564716249704361, "p95": 0.3817881643772125 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0745112895965576, "group": 2, "latest_source_mass": 0.412794291973114, "layer": 6, "max_source_mass": 0.412794291973114, "mixer_index": 10, "normalized_entropy": 0.9780623252441762, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.34262996912002563, "median": 0.3412924110889435, "p05": 0.29556700587272644, "p95": 0.39125555753707886 }, { "label": "completed_group_1", "mean": 0.24457570910453796, "median": 0.24382859468460083, "p05": 0.22120657563209534, "p95": 0.2734297513961792 }, { "label": "current_group_partial", "mean": 0.412794291973114, "median": 0.4151499271392822, "p05": 0.37735897302627563, "p95": 0.43749260902404785 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0074045658111572, "group": 2, "latest_source_mass": 0.3775783181190491, "layer": 6, "max_source_mass": 0.4399106502532959, "mixer_index": 11, "normalized_entropy": 0.9169791528842925, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.4399106502532959, "median": 0.44856905937194824, "p05": 0.22063353657722473, "p95": 0.6359403133392334 }, { "label": "completed_group_1", "mean": 0.18251106142997742, "median": 0.18562838435173035, "p05": 0.1196163073182106, "p95": 0.23045453429222107 }, { "label": "current_group_partial", "mean": 0.3775783181190491, "median": 0.3675365447998047, "p05": 0.23817461729049683, "p95": 0.5557451844215393 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.060497522354126, "group": 2, "latest_source_mass": 0.4025024175643921, "layer": 7, "max_source_mass": 0.4025024175643921, "mixer_index": 12, "normalized_entropy": 0.9653064445872968, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.3786153197288513, "median": 0.3907395303249359, "p05": 0.28039106726646423, "p95": 0.4436405599117279 }, { "label": "completed_group_1", "mean": 0.2188822478055954, "median": 0.21226292848587036, "p05": 0.1885872334241867, "p95": 0.2672030031681061 }, { "label": "current_group_partial", "mean": 0.4025024175643921, "median": 0.39815032482147217, "p05": 0.3533596992492676, "p95": 0.460100919008255 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 0.9941019415855408, "group": 2, "latest_source_mass": 0.4801652431488037, "layer": 7, "max_source_mass": 0.4801652431488037, "mixer_index": 13, "normalized_entropy": 0.9048705824970601, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.32296955585479736, "median": 0.2803739905357361, "p05": 0.1784922033548355, "p95": 0.6440245509147644 }, { "label": "completed_group_1", "mean": 0.19686520099639893, "median": 0.2053825855255127, "p05": 0.09813719242811203, "p95": 0.2504696547985077 }, { "label": "current_group_partial", "mean": 0.4801652431488037, "median": 0.5057140588760376, "p05": 0.2529003322124481, "p95": 0.598870575428009 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0503522157669067, "group": 2, "latest_source_mass": 0.4001518487930298, "layer": 8, "max_source_mass": 0.40130966901779175, "mixer_index": 14, "normalized_entropy": 0.9560717885654542, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.40130966901779175, "median": 0.396759569644928, "p05": 0.3479611873626709, "p95": 0.4733521044254303 }, { "label": "completed_group_1", "mean": 0.19853851199150085, "median": 0.19769755005836487, "p05": 0.17136667668819427, "p95": 0.22788912057876587 }, { "label": "current_group_partial", "mean": 0.4001518487930298, "median": 0.40291696786880493, "p05": 0.3506356179714203, "p95": 0.43941178917884827 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0129402875900269, "group": 2, "latest_source_mass": 0.42161279916763306, "layer": 8, "max_source_mass": 0.42161279916763306, "mixer_index": 15, "normalized_entropy": 0.9220179839951123, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.4093267023563385, "median": 0.4046899676322937, "p05": 0.2895161807537079, "p95": 0.5762131810188293 }, { "label": "completed_group_1", "mean": 0.16906049847602844, "median": 0.16888955235481262, "p05": 0.11233551055192947, "p95": 0.22477403283119202 }, { "label": "current_group_partial", "mean": 0.42161279916763306, "median": 0.4227708578109741, "p05": 0.3131049573421478, "p95": 0.519075334072113 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0774019956588745, "group": 3, "latest_source_mass": 0.3560434877872467, "layer": 9, "max_source_mass": 0.38510820269584656, "mixer_index": 16, "normalized_entropy": 0.9806935592947451, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.38510820269584656, "median": 0.3892287313938141, "p05": 0.2934730052947998, "p95": 0.4718902111053467 }, { "label": "completed_group_1", "mean": 0.25884830951690674, "median": 0.25734007358551025, "p05": 0.22217319905757904, "p95": 0.29551801085472107 }, { "label": "completed_group_2", "mean": 0.3560434877872467, "median": 0.35412120819091797, "p05": 0.29444968700408936, "p95": 0.41908568143844604 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.2740377187728882, "group": 3, "latest_source_mass": 0.3633861243724823, "layer": 9, "max_source_mass": 0.3633861243724823, "mixer_index": 17, "normalized_entropy": 0.9190239493895668, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.29550763964653015, "median": 0.30360478162765503, "p05": 0.16904757916927338, "p95": 0.45146965980529785 }, { "label": "completed_group_1", "mean": 0.09541244804859161, "median": 0.09486871212720871, "p05": 0.061360105872154236, "p95": 0.13037073612213135 }, { "label": "completed_group_2", "mean": 0.24569380283355713, "median": 0.24333015084266663, "p05": 0.1698218435049057, "p95": 0.31589236855506897 }, { "label": "current_group_partial", "mean": 0.3633861243724823, "median": 0.36472851037979126, "p05": 0.2795379161834717, "p95": 0.4399876892566681 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.360302448272705, "group": 3, "latest_source_mass": 0.28569501638412476, "layer": 10, "max_source_mass": 0.3109527826309204, "mixer_index": 18, "normalized_entropy": 0.9812507981160736, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.3109527826309204, "median": 0.32172977924346924, "p05": 0.23520725965499878, "p95": 0.37052470445632935 }, { "label": "completed_group_1", "mean": 0.20984819531440735, "median": 0.20958667993545532, "p05": 0.1891367882490158, "p95": 0.23184512555599213 }, { "label": "completed_group_2", "mean": 0.19350400567054749, "median": 0.19153150916099548, "p05": 0.16607457399368286, "p95": 0.22877082228660583 }, { "label": "current_group_partial", "mean": 0.28569501638412476, "median": 0.28060320019721985, "p05": 0.23824527859687805, "p95": 0.3439165949821472 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.2722704410552979, "group": 3, "latest_source_mass": 0.29912811517715454, "layer": 10, "max_source_mass": 0.35036465525627136, "mixer_index": 19, "normalized_entropy": 0.9177491279900463, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.35036465525627136, "median": 0.36210548877716064, "p05": 0.12807878851890564, "p95": 0.5797606706619263 }, { "label": "completed_group_1", "mean": 0.12631990015506744, "median": 0.1264108121395111, "p05": 0.06977735459804535, "p95": 0.1755298376083374 }, { "label": "completed_group_2", "mean": 0.22418735921382904, "median": 0.22563719749450684, "p05": 0.1176326721906662, "p95": 0.32001155614852905 }, { "label": "current_group_partial", "mean": 0.29912811517715454, "median": 0.28589946031570435, "p05": 0.193109393119812, "p95": 0.44119665026664734 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3558036088943481, "group": 3, "latest_source_mass": 0.314197838306427, "layer": 11, "max_source_mass": 0.314197838306427, "mixer_index": 20, "normalized_entropy": 0.978005571485618, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.298976868391037, "median": 0.2961939573287964, "p05": 0.24950163066387177, "p95": 0.35835936665534973 }, { "label": "completed_group_1", "mean": 0.1941717267036438, "median": 0.1918032318353653, "p05": 0.16809208691120148, "p95": 0.22895419597625732 }, { "label": "completed_group_2", "mean": 0.1926535815000534, "median": 0.19306811690330505, "p05": 0.16983944177627563, "p95": 0.2155935913324356 }, { "label": "current_group_partial", "mean": 0.314197838306427, "median": 0.31541913747787476, "p05": 0.2673022747039795, "p95": 0.35941022634506226 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.2696614265441895, "group": 3, "latest_source_mass": 0.2862822115421295, "layer": 11, "max_source_mass": 0.3667730689048767, "mixer_index": 21, "normalized_entropy": 0.9158671218416545, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.3667730689048767, "median": 0.4067659378051758, "p05": 0.10593366622924805, "p95": 0.5672274827957153 }, { "label": "completed_group_1", "mean": 0.15310388803482056, "median": 0.15050218999385834, "p05": 0.08651046454906464, "p95": 0.22459912300109863 }, { "label": "completed_group_2", "mean": 0.19384080171585083, "median": 0.18732509016990662, "p05": 0.1267009675502777, "p95": 0.281107634305954 }, { "label": "current_group_partial", "mean": 0.2862822115421295, "median": 0.26271265745162964, "p05": 0.17024393379688263, "p95": 0.475044310092926 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3606936931610107, "group": 3, "latest_source_mass": 0.3028184771537781, "layer": 12, "max_source_mass": 0.3028184771537781, "mixer_index": 22, "normalized_entropy": 0.9815330216461395, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.298227459192276, "median": 0.2954302132129669, "p05": 0.25287362933158875, "p95": 0.35405808687210083 }, { "label": "completed_group_1", "mean": 0.18611131608486176, "median": 0.1850002110004425, "p05": 0.15901757776737213, "p95": 0.21725836396217346 }, { "label": "completed_group_2", "mean": 0.21284276247024536, "median": 0.21402311325073242, "p05": 0.18819917738437653, "p95": 0.23418807983398438 }, { "label": "current_group_partial", "mean": 0.3028184771537781, "median": 0.3041122555732727, "p05": 0.25661808252334595, "p95": 0.34458184242248535 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.269316554069519, "group": 3, "latest_source_mass": 0.2653314769268036, "layer": 12, "max_source_mass": 0.4081852436065674, "mixer_index": 23, "normalized_entropy": 0.9156183489371815, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.4081852436065674, "median": 0.4254342317581177, "p05": 0.20765496790409088, "p95": 0.587238073348999 }, { "label": "completed_group_1", "mean": 0.14420127868652344, "median": 0.1438048928976059, "p05": 0.1064920499920845, "p95": 0.1871720254421234 }, { "label": "completed_group_2", "mean": 0.18228203058242798, "median": 0.17507661879062653, "p05": 0.12183455377817154, "p95": 0.26505547761917114 }, { "label": "current_group_partial", "mean": 0.2653314769268036, "median": 0.24924281239509583, "p05": 0.15609969198703766, "p95": 0.4061311185359955 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3636136054992676, "group": 4, "latest_source_mass": 0.3136237859725952, "layer": 13, "max_source_mass": 0.3136237859725952, "mixer_index": 24, "normalized_entropy": 0.9836392931712563, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.27427661418914795, "median": 0.27403637766838074, "p05": 0.22928935289382935, "p95": 0.319773405790329 }, { "label": "completed_group_1", "mean": 0.21341991424560547, "median": 0.20697325468063354, "p05": 0.18552586436271667, "p95": 0.26688331365585327 }, { "label": "completed_group_2", "mean": 0.19867971539497375, "median": 0.197545126080513, "p05": 0.1794913411140442, "p95": 0.222100168466568 }, { "label": "completed_group_3", "mean": 0.3136237859725952, "median": 0.31875866651535034, "p05": 0.24156072735786438, "p95": 0.3695686459541321 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.570773959159851, "group": 4, "latest_source_mass": 0.2294243425130844, "layer": 13, "max_source_mass": 0.2658740282058716, "mixer_index": 25, "normalized_entropy": 0.9759767351225285, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.2658740282058716, "median": 0.2747163474559784, "p05": 0.17927880585193634, "p95": 0.3512483835220337 }, { "label": "completed_group_1", "mean": 0.13419361412525177, "median": 0.13374894857406616, "p05": 0.11072897166013718, "p95": 0.1586270034313202 }, { "label": "completed_group_2", "mean": 0.1907704919576645, "median": 0.19074805080890656, "p05": 0.14699944853782654, "p95": 0.23316305875778198 }, { "label": "completed_group_3", "mean": 0.17973749339580536, "median": 0.17616114020347595, "p05": 0.1363743096590042, "p95": 0.23202598094940186 }, { "label": "current_group_partial", "mean": 0.2294243425130844, "median": 0.22995705902576447, "p05": 0.17975632846355438, "p95": 0.2789157032966614 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.5560038089752197, "group": 4, "latest_source_mass": 0.32468360662460327, "layer": 14, "max_source_mass": 0.32468360662460327, "mixer_index": 26, "normalized_entropy": 0.9667995248241249, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.20477107167243958, "median": 0.206403911113739, "p05": 0.16619175672531128, "p95": 0.2431035190820694 }, { "label": "completed_group_1", "mean": 0.15721669793128967, "median": 0.15658551454544067, "p05": 0.13878680765628815, "p95": 0.17778633534908295 }, { "label": "completed_group_2", "mean": 0.13811248540878296, "median": 0.13743364810943604, "p05": 0.12352626770734787, "p95": 0.15553249418735504 }, { "label": "completed_group_3", "mean": 0.1752161681652069, "median": 0.17309701442718506, "p05": 0.15223677456378937, "p95": 0.20474471151828766 }, { "label": "current_group_partial", "mean": 0.32468360662460327, "median": 0.32497870922088623, "p05": 0.2946181893348694, "p95": 0.35203880071640015 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.5659973621368408, "group": 4, "latest_source_mass": 0.26703086495399475, "layer": 14, "max_source_mass": 0.26703086495399475, "mixer_index": 27, "normalized_entropy": 0.9730088685238187, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.20531430840492249, "median": 0.20229148864746094, "p05": 0.10698921978473663, "p95": 0.3393132984638214 }, { "label": "completed_group_1", "mean": 0.15986156463623047, "median": 0.15952220559120178, "p05": 0.11969029903411865, "p95": 0.20180393755435944 }, { "label": "completed_group_2", "mean": 0.21557100117206573, "median": 0.21654027700424194, "p05": 0.1673085242509842, "p95": 0.2610440254211426 }, { "label": "completed_group_3", "mean": 0.15222224593162537, "median": 0.1510905921459198, "p05": 0.11800260841846466, "p95": 0.18893754482269287 }, { "label": "current_group_partial", "mean": 0.26703086495399475, "median": 0.258539617061615, "p05": 0.19330690801143646, "p95": 0.36521556973457336 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.5552496910095215, "group": 4, "latest_source_mass": 0.31774693727493286, "layer": 15, "max_source_mass": 0.31774693727493286, "mixer_index": 28, "normalized_entropy": 0.9663309649872576, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.2162160575389862, "median": 0.2145705670118332, "p05": 0.17727568745613098, "p95": 0.26878347992897034 }, { "label": "completed_group_1", "mean": 0.17982497811317444, "median": 0.18024154007434845, "p05": 0.15453973412513733, "p95": 0.2030555009841919 }, { "label": "completed_group_2", "mean": 0.1307673156261444, "median": 0.13107559084892273, "p05": 0.11463287472724915, "p95": 0.14630235731601715 }, { "label": "completed_group_3", "mean": 0.1554446816444397, "median": 0.15516987442970276, "p05": 0.13627474009990692, "p95": 0.1754152774810791 }, { "label": "current_group_partial", "mean": 0.31774693727493286, "median": 0.319039523601532, "p05": 0.27606821060180664, "p95": 0.35650834441185 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.5068919658660889, "group": 4, "latest_source_mass": 0.3298327624797821, "layer": 15, "max_source_mass": 0.3298327624797821, "mixer_index": 29, "normalized_entropy": 0.9362846209998111, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.20646697282791138, "median": 0.18775638937950134, "p05": 0.0806753933429718, "p95": 0.4274584949016571 }, { "label": "completed_group_1", "mean": 0.1197461187839508, "median": 0.11989086866378784, "p05": 0.0809398666024208, "p95": 0.15743419528007507 }, { "label": "completed_group_2", "mean": 0.19215428829193115, "median": 0.19620640575885773, "p05": 0.1148109883069992, "p95": 0.2491212636232376 }, { "label": "completed_group_3", "mean": 0.15179988741874695, "median": 0.15212899446487427, "p05": 0.10697861760854721, "p95": 0.19542087614536285 }, { "label": "current_group_partial", "mean": 0.3298327624797821, "median": 0.32837340235710144, "p05": 0.22746729850769043, "p95": 0.4339301884174347 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.5435223579406738, "group": 4, "latest_source_mass": 0.3224271535873413, "layer": 16, "max_source_mass": 0.3224271535873413, "mixer_index": 30, "normalized_entropy": 0.9590443632623663, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.22410975396633148, "median": 0.22760415077209473, "p05": 0.14593973755836487, "p95": 0.30093055963516235 }, { "label": "completed_group_1", "mean": 0.16986176371574402, "median": 0.16963240504264832, "p05": 0.13569067418575287, "p95": 0.20651353895664215 }, { "label": "completed_group_2", "mean": 0.13080403208732605, "median": 0.13075244426727295, "p05": 0.10945598781108856, "p95": 0.15187400579452515 }, { "label": "completed_group_3", "mean": 0.15279728174209595, "median": 0.15036612749099731, "p05": 0.12810495495796204, "p95": 0.18468116223812103 }, { "label": "current_group_partial", "mean": 0.3224271535873413, "median": 0.3164973258972168, "p05": 0.2600978910923004, "p95": 0.40530598163604736 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.4949920177459717, "group": 4, "latest_source_mass": 0.31847020983695984, "layer": 16, "max_source_mass": 0.31847020983695984, "mixer_index": 31, "normalized_entropy": 0.9288907675133354, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.27174079418182373, "median": 0.2585451304912567, "p05": 0.13639721274375916, "p95": 0.4698348045349121 }, { "label": "completed_group_1", "mean": 0.12178683280944824, "median": 0.11714950203895569, "p05": 0.08594125509262085, "p95": 0.1723237782716751 }, { "label": "completed_group_2", "mean": 0.1529826819896698, "median": 0.15223106741905212, "p05": 0.10371473431587219, "p95": 0.203575000166893 }, { "label": "completed_group_3", "mean": 0.1350194811820984, "median": 0.13363957405090332, "p05": 0.08964549005031586, "p95": 0.18468207120895386 }, { "label": "current_group_partial", "mean": 0.31847020983695984, "median": 0.3185216188430786, "p05": 0.21444711089134216, "p95": 0.4301546812057495 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.544264793395996, "group": 5, "latest_source_mass": 0.2861037850379944, "layer": 17, "max_source_mass": 0.2861037850379944, "mixer_index": 32, "normalized_entropy": 0.9595056643474137, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.24016115069389343, "median": 0.23868846893310547, "p05": 0.18435582518577576, "p95": 0.2983453571796417 }, { "label": "completed_group_1", "mean": 0.20730282366275787, "median": 0.1998077780008316, "p05": 0.16255050897598267, "p95": 0.28107208013534546 }, { "label": "completed_group_2", "mean": 0.09692573547363281, "median": 0.0967404842376709, "p05": 0.08098706603050232, "p95": 0.11221462488174438 }, { "label": "completed_group_3", "mean": 0.1695065200328827, "median": 0.1696585714817047, "p05": 0.136396586894989, "p95": 0.20130634307861328 }, { "label": "completed_group_4", "mean": 0.2861037850379944, "median": 0.28883200883865356, "p05": 0.22401469945907593, "p95": 0.3409370183944702 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.686682939529419, "group": 5, "latest_source_mass": 0.12281867861747742, "layer": 17, "max_source_mass": 0.2885604202747345, "mixer_index": 33, "normalized_entropy": 0.9413556721740635, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.16755425930023193, "median": 0.1327776312828064, "p05": 0.0790192037820816, "p95": 0.3667412996292114 }, { "label": "completed_group_1", "mean": 0.10962115228176117, "median": 0.10525335371494293, "p05": 0.07840172946453094, "p95": 0.15405339002609253 }, { "label": "completed_group_2", "mean": 0.1783454418182373, "median": 0.17896884679794312, "p05": 0.10802385956048965, "p95": 0.24048911035060883 }, { "label": "completed_group_3", "mean": 0.13310004770755768, "median": 0.13033035397529602, "p05": 0.0904831737279892, "p95": 0.17934636771678925 }, { "label": "completed_group_4", "mean": 0.2885604202747345, "median": 0.2831825315952301, "p05": 0.16967125236988068, "p95": 0.4189499616622925 }, { "label": "current_group_partial", "mean": 0.12281867861747742, "median": 0.12062528729438782, "p05": 0.07461707293987274, "p95": 0.1782219111919403 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.7070834636688232, "group": 5, "latest_source_mass": 0.2988377809524536, "layer": 18, "max_source_mass": 0.2988377809524536, "mixer_index": 34, "normalized_entropy": 0.9527414214834803, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.181177020072937, "median": 0.18419909477233887, "p05": 0.13624237477779388, "p95": 0.22192169725894928 }, { "label": "completed_group_1", "mean": 0.14557164907455444, "median": 0.144476979970932, "p05": 0.1293315887451172, "p95": 0.16486899554729462 }, { "label": "completed_group_2", "mean": 0.08318156749010086, "median": 0.0820261612534523, "p05": 0.07088497281074524, "p95": 0.0990293100476265 }, { "label": "completed_group_3", "mean": 0.11424209177494049, "median": 0.11267061531543732, "p05": 0.0979941189289093, "p95": 0.1353992223739624 }, { "label": "completed_group_4", "mean": 0.17698991298675537, "median": 0.17597860097885132, "p05": 0.14837387204170227, "p95": 0.21050144731998444 }, { "label": "current_group_partial", "mean": 0.2988377809524536, "median": 0.2989804744720459, "p05": 0.2721370458602905, "p95": 0.32572072744369507 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.7419235706329346, "group": 5, "latest_source_mass": 0.26016175746917725, "layer": 18, "max_source_mass": 0.26016175746917725, "mixer_index": 35, "normalized_entropy": 0.972186055410333, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1532188355922699, "median": 0.1451343297958374, "p05": 0.11514341086149216, "p95": 0.22103679180145264 }, { "label": "completed_group_1", "mean": 0.11420352011919022, "median": 0.11261284351348877, "p05": 0.0928458571434021, "p95": 0.14123988151550293 }, { "label": "completed_group_2", "mean": 0.16597424447536469, "median": 0.1655328869819641, "p05": 0.13871800899505615, "p95": 0.19611573219299316 }, { "label": "completed_group_3", "mean": 0.1250365972518921, "median": 0.12369821965694427, "p05": 0.10375828295946121, "p95": 0.14912742376327515 }, { "label": "completed_group_4", "mean": 0.18140508234500885, "median": 0.1796584278345108, "p05": 0.14511024951934814, "p95": 0.22286373376846313 }, { "label": "current_group_partial", "mean": 0.26016175746917725, "median": 0.26266711950302124, "p05": 0.1889389455318451, "p95": 0.3234381675720215 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.7127439975738525, "group": 5, "latest_source_mass": 0.2907913327217102, "layer": 19, "max_source_mass": 0.2907913327217102, "mixer_index": 36, "normalized_entropy": 0.9559006256078307, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.18731768429279327, "median": 0.19042012095451355, "p05": 0.13879702985286713, "p95": 0.23792000114917755 }, { "label": "completed_group_1", "mean": 0.16715484857559204, "median": 0.1667211651802063, "p05": 0.14574480056762695, "p95": 0.19026139378547668 }, { "label": "completed_group_2", "mean": 0.08496320247650146, "median": 0.08452782034873962, "p05": 0.07410907745361328, "p95": 0.097386933863163 }, { "label": "completed_group_3", "mean": 0.11362075805664062, "median": 0.11248823255300522, "p05": 0.0977015346288681, "p95": 0.13219426572322845 }, { "label": "completed_group_4", "mean": 0.15615218877792358, "median": 0.15540224313735962, "p05": 0.1289147436618805, "p95": 0.18598109483718872 }, { "label": "current_group_partial", "mean": 0.2907913327217102, "median": 0.2900415062904358, "p05": 0.25822532176971436, "p95": 0.32475030422210693 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.6076834201812744, "group": 5, "latest_source_mass": 0.3738824725151062, "layer": 19, "max_source_mass": 0.3738824725151062, "mixer_index": 37, "normalized_entropy": 0.8972652009334232, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.15032559633255005, "median": 0.12544110417366028, "p05": 0.06976518034934998, "p95": 0.35840943455696106 }, { "label": "completed_group_1", "mean": 0.08624927699565887, "median": 0.08602266013622284, "p05": 0.061998721212148666, "p95": 0.11138717085123062 }, { "label": "completed_group_2", "mean": 0.15736976265907288, "median": 0.15802210569381714, "p05": 0.10121746361255646, "p95": 0.20687000453472137 }, { "label": "completed_group_3", "mean": 0.08584291487932205, "median": 0.0827617198228836, "p05": 0.060561805963516235, "p95": 0.11351694166660309 }, { "label": "completed_group_4", "mean": 0.14632999897003174, "median": 0.1426895558834076, "p05": 0.10106660425662994, "p95": 0.20179018378257751 }, { "label": "current_group_partial", "mean": 0.3738824725151062, "median": 0.3799295425415039, "p05": 0.23631010949611664, "p95": 0.4884527325630188 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.709290862083435, "group": 5, "latest_source_mass": 0.2901708781719208, "layer": 20, "max_source_mass": 0.2901708781719208, "mixer_index": 38, "normalized_entropy": 0.9539733939957076, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.18991519510746002, "median": 0.1924753338098526, "p05": 0.14201581478118896, "p95": 0.23709386587142944 }, { "label": "completed_group_1", "mean": 0.17091524600982666, "median": 0.17006167769432068, "p05": 0.15297070145606995, "p95": 0.19153237342834473 }, { "label": "completed_group_2", "mean": 0.07822999358177185, "median": 0.07778562605381012, "p05": 0.06707640737295151, "p95": 0.09089364111423492 }, { "label": "completed_group_3", "mean": 0.116765096783638, "median": 0.11516137421131134, "p05": 0.10162825882434845, "p95": 0.13653048872947693 }, { "label": "completed_group_4", "mean": 0.1540035903453827, "median": 0.15287795662879944, "p05": 0.12593276798725128, "p95": 0.18590238690376282 }, { "label": "current_group_partial", "mean": 0.2901708781719208, "median": 0.29104921221733093, "p05": 0.25643545389175415, "p95": 0.3205650746822357 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.6690887212753296, "group": 5, "latest_source_mass": 0.3342471718788147, "layer": 20, "max_source_mass": 0.3342471718788147, "mixer_index": 39, "normalized_entropy": 0.9315361520005944, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.19029226899147034, "median": 0.19644704461097717, "p05": 0.09456793963909149, "p95": 0.30468082427978516 }, { "label": "completed_group_1", "mean": 0.1100241169333458, "median": 0.10808776319026947, "p05": 0.09035106748342514, "p95": 0.1359245330095291 }, { "label": "completed_group_2", "mean": 0.11471512168645859, "median": 0.11494918912649155, "p05": 0.08799737691879272, "p95": 0.14024731516838074 }, { "label": "completed_group_3", "mean": 0.09999909996986389, "median": 0.09682698547840118, "p05": 0.07921088486909866, "p95": 0.1264391839504242 }, { "label": "completed_group_4", "mean": 0.1507222205400467, "median": 0.1476232260465622, "p05": 0.11397356539964676, "p95": 0.19991227984428406 }, { "label": "current_group_partial", "mean": 0.3342471718788147, "median": 0.33802270889282227, "p05": 0.2548266053199768, "p95": 0.40130624175071716 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.7458693981170654, "group": 6, "latest_source_mass": 0.2361394464969635, "layer": 21, "max_source_mass": 0.2361394464969635, "mixer_index": 40, "normalized_entropy": 0.9743882636597644, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.20528709888458252, "median": 0.2140858769416809, "p05": 0.13168928027153015, "p95": 0.27098023891448975 }, { "label": "completed_group_1", "mean": 0.1722964197397232, "median": 0.17238952219486237, "p05": 0.15053534507751465, "p95": 0.19365043938159943 }, { "label": "completed_group_2", "mean": 0.10435444861650467, "median": 0.10361101478338242, "p05": 0.08888213336467743, "p95": 0.12212536484003067 }, { "label": "completed_group_3", "mean": 0.12075909972190857, "median": 0.1180175170302391, "p05": 0.10358668118715286, "p95": 0.14458929002285004 }, { "label": "completed_group_4", "mean": 0.16116347908973694, "median": 0.15841388702392578, "p05": 0.1337081342935562, "p95": 0.19791515171527863 }, { "label": "completed_group_5", "mean": 0.2361394464969635, "median": 0.23709020018577576, "p05": 0.20446214079856873, "p95": 0.26509594917297363 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.4612125158309937, "group": 6, "latest_source_mass": 0.5543455481529236, "layer": 21, "max_source_mass": 0.5543455481529236, "mixer_index": 41, "normalized_entropy": 0.7509146897354808, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.08934466540813446, "median": 0.07911523431539536, "p05": 0.04488275945186615, "p95": 0.1737232506275177 }, { "label": "completed_group_1", "mean": 0.054379627108573914, "median": 0.05344422906637192, "p05": 0.04221750423312187, "p95": 0.06924191862344742 }, { "label": "completed_group_2", "mean": 0.07567165046930313, "median": 0.07438549399375916, "p05": 0.05811355262994766, "p95": 0.09632256627082825 }, { "label": "completed_group_3", "mean": 0.05758609622716904, "median": 0.05612001568078995, "p05": 0.044792283326387405, "p95": 0.07347827404737473 }, { "label": "completed_group_4", "mean": 0.07884950935840607, "median": 0.07757881283760071, "p05": 0.05758974328637123, "p95": 0.10452556610107422 }, { "label": "completed_group_5", "mean": 0.08982288837432861, "median": 0.08912864327430725, "p05": 0.0663854256272316, "p95": 0.11604885756969452 }, { "label": "current_group_partial", "mean": 0.5543455481529236, "median": 0.5596089363098145, "p05": 0.45455610752105713, "p95": 0.6367484331130981 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.7241814136505127, "group": 6, "latest_source_mass": 0.39484071731567383, "layer": 22, "max_source_mass": 0.39484071731567383, "mixer_index": 42, "normalized_entropy": 0.886053970419732, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.13360010087490082, "median": 0.14072942733764648, "p05": 0.07699814438819885, "p95": 0.17506597936153412 }, { "label": "completed_group_1", "mean": 0.10609637200832367, "median": 0.10586297512054443, "p05": 0.08648421615362167, "p95": 0.12602579593658447 }, { "label": "completed_group_2", "mean": 0.06363911926746368, "median": 0.06324908882379532, "p05": 0.054980404675006866, "p95": 0.07316673547029495 }, { "label": "completed_group_3", "mean": 0.07426095008850098, "median": 0.07362888753414154, "p05": 0.06318790465593338, "p95": 0.08665015548467636 }, { "label": "completed_group_4", "mean": 0.09383076429367065, "median": 0.09294568747282028, "p05": 0.07638302445411682, "p95": 0.11445100605487823 }, { "label": "completed_group_5", "mean": 0.13373197615146637, "median": 0.13327741622924805, "p05": 0.11399447172880173, "p95": 0.15641780197620392 }, { "label": "current_group_partial", "mean": 0.39484071731567383, "median": 0.3879151940345764, "p05": 0.3302172124385834, "p95": 0.4852263331413269 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.3979300260543823, "group": 6, "latest_source_mass": 0.5830056071281433, "layer": 22, "max_source_mass": 0.5830056071281433, "mixer_index": 43, "normalized_entropy": 0.7183939231382495, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.08959861099720001, "median": 0.07544538378715515, "p05": 0.04165133833885193, "p95": 0.20099139213562012 }, { "label": "completed_group_1", "mean": 0.05414731800556183, "median": 0.05336771905422211, "p05": 0.04069708660244942, "p95": 0.07046990841627121 }, { "label": "completed_group_2", "mean": 0.07668881118297577, "median": 0.07607124000787735, "p05": 0.0555989071726799, "p95": 0.09980960935354233 }, { "label": "completed_group_3", "mean": 0.05603286623954773, "median": 0.054573021829128265, "p05": 0.04162490367889404, "p95": 0.0730118378996849 }, { "label": "completed_group_4", "mean": 0.06979228556156158, "median": 0.06884932518005371, "p05": 0.04576495662331581, "p95": 0.09806846082210541 }, { "label": "completed_group_5", "mean": 0.07073447108268738, "median": 0.07023533433675766, "p05": 0.04956750199198723, "p95": 0.09379896521568298 }, { "label": "current_group_partial", "mean": 0.5830056071281433, "median": 0.5862616896629333, "p05": 0.4658474922180176, "p95": 0.6849814057350159 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.7459560632705688, "group": 6, "latest_source_mass": 0.37691548466682434, "layer": 23, "max_source_mass": 0.37691548466682434, "mixer_index": 44, "normalized_entropy": 0.897243926765161, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.14318783581256866, "median": 0.1493408977985382, "p05": 0.08292607218027115, "p95": 0.19464610517024994 }, { "label": "completed_group_1", "mean": 0.1094653457403183, "median": 0.10858799517154694, "p05": 0.09432064741849899, "p95": 0.1279931515455246 }, { "label": "completed_group_2", "mean": 0.06349228322505951, "median": 0.06325754523277283, "p05": 0.05439673364162445, "p95": 0.0737837627530098 }, { "label": "completed_group_3", "mean": 0.07859592139720917, "median": 0.0775163471698761, "p05": 0.06603655964136124, "p95": 0.09357087314128876 }, { "label": "completed_group_4", "mean": 0.09567482024431229, "median": 0.09468592703342438, "p05": 0.07671141624450684, "p95": 0.11774884164333344 }, { "label": "completed_group_5", "mean": 0.13266831636428833, "median": 0.13199329376220703, "p05": 0.11244359612464905, "p95": 0.15504513680934906 }, { "label": "current_group_partial", "mean": 0.37691548466682434, "median": 0.37180638313293457, "p05": 0.31207239627838135, "p95": 0.4595233201980591 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.4055393934249878, "group": 6, "latest_source_mass": 0.5772827863693237, "layer": 23, "max_source_mass": 0.5772827863693237, "mixer_index": 45, "normalized_entropy": 0.7223043644164862, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.10154955089092255, "median": 0.08766824007034302, "p05": 0.04901552200317383, "p95": 0.20287738740444183 }, { "label": "completed_group_1", "mean": 0.05619276314973831, "median": 0.055477917194366455, "p05": 0.041608311235904694, "p95": 0.07371112704277039 }, { "label": "completed_group_2", "mean": 0.07791627198457718, "median": 0.07736755907535553, "p05": 0.055920060724020004, "p95": 0.10158214718103409 }, { "label": "completed_group_3", "mean": 0.053915880620479584, "median": 0.05215863138437271, "p05": 0.03798118978738785, "p95": 0.07156119495630264 }, { "label": "completed_group_4", "mean": 0.06398355215787888, "median": 0.06229078769683838, "p05": 0.040616974234580994, "p95": 0.09270931780338287 }, { "label": "completed_group_5", "mean": 0.06915920227766037, "median": 0.0684593990445137, "p05": 0.046986356377601624, "p95": 0.09408729523420334 }, { "label": "current_group_partial", "mean": 0.5772827863693237, "median": 0.5824689269065857, "p05": 0.4518115520477295, "p95": 0.6926202178001404 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.7853620052337646, "group": 6, "latest_source_mass": 0.3407441973686218, "layer": 24, "max_source_mass": 0.3407441973686218, "mixer_index": 46, "normalized_entropy": 0.9174945750195659, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.15284113585948944, "median": 0.1593906283378601, "p05": 0.08678612112998962, "p95": 0.211342453956604 }, { "label": "completed_group_1", "mean": 0.12206703424453735, "median": 0.12153112143278122, "p05": 0.1060115247964859, "p95": 0.1399766057729721 }, { "label": "completed_group_2", "mean": 0.06508444249629974, "median": 0.06482555717229843, "p05": 0.055350616574287415, "p95": 0.07557765394449234 }, { "label": "completed_group_3", "mean": 0.08347313106060028, "median": 0.08190013468265533, "p05": 0.06966532766819, "p95": 0.10050469636917114 }, { "label": "completed_group_4", "mean": 0.09811970591545105, "median": 0.09664031863212585, "p05": 0.07688035070896149, "p95": 0.12369741499423981 }, { "label": "completed_group_5", "mean": 0.13767032325267792, "median": 0.1373325139284134, "p05": 0.11642646044492722, "p95": 0.1606694608926773 }, { "label": "current_group_partial", "mean": 0.3407441973686218, "median": 0.33805108070373535, "p05": 0.271301805973053, "p95": 0.41781002283096313 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.509842872619629, "group": 6, "latest_source_mass": 0.5227814316749573, "layer": 24, "max_source_mass": 0.5227814316749573, "mixer_index": 47, "normalized_entropy": 0.7759057494780099, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.11788781732320786, "median": 0.10097194463014603, "p05": 0.06265684217214584, "p95": 0.2408325970172882 }, { "label": "completed_group_1", "mean": 0.06822571158409119, "median": 0.0673636943101883, "p05": 0.05209728330373764, "p95": 0.08648915588855743 }, { "label": "completed_group_2", "mean": 0.08556114137172699, "median": 0.08463940769433975, "p05": 0.06234566867351532, "p95": 0.1125786080956459 }, { "label": "completed_group_3", "mean": 0.06067090481519699, "median": 0.058974385261535645, "p05": 0.043267179280519485, "p95": 0.07997901737689972 }, { "label": "completed_group_4", "mean": 0.07228855788707733, "median": 0.06972275674343109, "p05": 0.04689513146877289, "p95": 0.106512151658535 }, { "label": "completed_group_5", "mean": 0.07258446514606476, "median": 0.07082539796829224, "p05": 0.050949547439813614, "p95": 0.09970614314079285 }, { "label": "current_group_partial", "mean": 0.5227814316749573, "median": 0.5290273427963257, "p05": 0.38773614168167114, "p95": 0.6392589807510376 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.839367389678955, "group": 7, "latest_source_mass": 0.29583215713500977, "layer": 25, "max_source_mass": 0.29583215713500977, "mixer_index": 48, "normalized_entropy": 0.9452478525649903, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.15906387567520142, "median": 0.15846745669841766, "p05": 0.09519976377487183, "p95": 0.24517810344696045 }, { "label": "completed_group_1", "mean": 0.13139380514621735, "median": 0.1308426856994629, "p05": 0.11478274315595627, "p95": 0.15000829100608826 }, { "label": "completed_group_2", "mean": 0.07717745751142502, "median": 0.07715500891208649, "p05": 0.06616044044494629, "p95": 0.08791667222976685 }, { "label": "completed_group_3", "mean": 0.09607303142547607, "median": 0.09436552226543427, "p05": 0.08152313530445099, "p95": 0.11384639143943787 }, { "label": "completed_group_4", "mean": 0.1107850894331932, "median": 0.1089654266834259, "p05": 0.08814431726932526, "p95": 0.139452263712883 }, { "label": "completed_group_5", "mean": 0.12967455387115479, "median": 0.1299237608909607, "p05": 0.10832332074642181, "p95": 0.15138982236385345 }, { "label": "completed_group_6", "mean": 0.29583215713500977, "median": 0.29647016525268555, "p05": 0.2276897132396698, "p95": 0.3604946434497833 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.5841834545135498, "group": 7, "latest_source_mass": 0.40767741203308105, "layer": 25, "max_source_mass": 0.40767741203308105, "mixer_index": 49, "normalized_entropy": 0.7618312045616817, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.07896879315376282, "median": 0.06809917092323303, "p05": 0.025932887569069862, "p95": 0.19368338584899902 }, { "label": "completed_group_1", "mean": 0.048752427101135254, "median": 0.04767637699842453, "p05": 0.034443892538547516, "p95": 0.0667153000831604 }, { "label": "completed_group_2", "mean": 0.055707164108753204, "median": 0.0549345538020134, "p05": 0.038696639239788055, "p95": 0.0750277042388916 }, { "label": "completed_group_3", "mean": 0.04139826446771622, "median": 0.03878612071275711, "p05": 0.026673054322600365, "p95": 0.05746251344680786 }, { "label": "completed_group_4", "mean": 0.04449018836021423, "median": 0.04196460545063019, "p05": 0.026162121444940567, "p95": 0.07164917886257172 }, { "label": "completed_group_5", "mean": 0.04285799711942673, "median": 0.04109475016593933, "p05": 0.029393533244729042, "p95": 0.06277710199356079 }, { "label": "completed_group_6", "mean": 0.2801477909088135, "median": 0.2763082981109619, "p05": 0.1455708146095276, "p95": 0.4254714846611023 }, { "label": "current_group_partial", "mean": 0.40767741203308105, "median": 0.4120897948741913, "p05": 0.21399420499801636, "p95": 0.5853031277656555 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.8125314712524414, "group": 7, "latest_source_mass": 0.3700707256793976, "layer": 26, "max_source_mass": 0.3700707256793976, "mixer_index": 50, "normalized_entropy": 0.8716433883436914, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.10889513790607452, "median": 0.11333495378494263, "p05": 0.04850132763385773, "p95": 0.1620434820652008 }, { "label": "completed_group_1", "mean": 0.08656036853790283, "median": 0.08640051633119583, "p05": 0.0650845319032669, "p95": 0.1084962785243988 }, { "label": "completed_group_2", "mean": 0.048123255372047424, "median": 0.04800213500857353, "p05": 0.03576180338859558, "p95": 0.06107594817876816 }, { "label": "completed_group_3", "mean": 0.06561170518398285, "median": 0.06379580497741699, "p05": 0.050968848168849945, "p95": 0.08332888036966324 }, { "label": "completed_group_4", "mean": 0.06917259097099304, "median": 0.06724178791046143, "p05": 0.050207968801259995, "p95": 0.09373342245817184 }, { "label": "completed_group_5", "mean": 0.08286689221858978, "median": 0.08183546364307404, "p05": 0.06535022705793381, "p95": 0.10340933501720428 }, { "label": "completed_group_6", "mean": 0.16869932413101196, "median": 0.16509854793548584, "p05": 0.11059704422950745, "p95": 0.23654375970363617 }, { "label": "current_group_partial", "mean": 0.3700707256793976, "median": 0.3694133758544922, "p05": 0.23441238701343536, "p95": 0.5025027990341187 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.50089693069458, "group": 7, "latest_source_mass": 0.49096518754959106, "layer": 26, "max_source_mass": 0.49096518754959106, "mixer_index": 51, "normalized_entropy": 0.7217788529328457, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.07974998652935028, "median": 0.07402504235506058, "p05": 0.04579866677522659, "p95": 0.1359037309885025 }, { "label": "completed_group_1", "mean": 0.04166398197412491, "median": 0.04038450866937637, "p05": 0.027992289513349533, "p95": 0.05969971418380737 }, { "label": "completed_group_2", "mean": 0.04607272148132324, "median": 0.044438350945711136, "p05": 0.030038591474294662, "p95": 0.06808309257030487 }, { "label": "completed_group_3", "mean": 0.03586764633655548, "median": 0.03317485377192497, "p05": 0.02203454077243805, "p95": 0.05572832003235817 }, { "label": "completed_group_4", "mean": 0.04223340004682541, "median": 0.03930673003196716, "p05": 0.02465122565627098, "p95": 0.06968094408512115 }, { "label": "completed_group_5", "mean": 0.04095568507909775, "median": 0.03919431194663048, "p05": 0.026239857077598572, "p95": 0.06058088690042496 }, { "label": "completed_group_6", "mean": 0.22249138355255127, "median": 0.2125401496887207, "p05": 0.10647299885749817, "p95": 0.3711881935596466 }, { "label": "current_group_partial", "mean": 0.49096518754959106, "median": 0.4988659620285034, "p05": 0.2878011167049408, "p95": 0.6676807403564453 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.8059929609298706, "group": 7, "latest_source_mass": 0.37139976024627686, "layer": 27, "max_source_mass": 0.37139976024627686, "mixer_index": 52, "normalized_entropy": 0.8684990295379666, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.10711923241615295, "median": 0.10972198843955994, "p05": 0.04896187782287598, "p95": 0.16201043128967285 }, { "label": "completed_group_1", "mean": 0.07651707530021667, "median": 0.07666032761335373, "p05": 0.05141128972172737, "p95": 0.10152198374271393 }, { "label": "completed_group_2", "mean": 0.058284737169742584, "median": 0.05812416970729828, "p05": 0.039273619651794434, "p95": 0.07745087146759033 }, { "label": "completed_group_3", "mean": 0.06609587371349335, "median": 0.06383530795574188, "p05": 0.04481930285692215, "p95": 0.09336265176534653 }, { "label": "completed_group_4", "mean": 0.07139429450035095, "median": 0.0694514587521553, "p05": 0.0485839769244194, "p95": 0.10034126043319702 }, { "label": "completed_group_5", "mean": 0.07493151724338531, "median": 0.07319007813930511, "p05": 0.051876723766326904, "p95": 0.10402432829141617 }, { "label": "completed_group_6", "mean": 0.17425751686096191, "median": 0.16937679052352905, "p05": 0.10449051856994629, "p95": 0.2624811828136444 }, { "label": "current_group_partial", "mean": 0.37139976024627686, "median": 0.3732905089855194, "p05": 0.23830299079418182, "p95": 0.5014542937278748 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.5622732639312744, "group": 7, "latest_source_mass": 0.4768889844417572, "layer": 27, "max_source_mass": 0.4768889844417572, "mixer_index": 53, "normalized_entropy": 0.7512946301290215, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.08775827288627625, "median": 0.08504067361354828, "p05": 0.0428270623087883, "p95": 0.144285649061203 }, { "label": "completed_group_1", "mean": 0.047765426337718964, "median": 0.04646281152963638, "p05": 0.031814731657505035, "p95": 0.06807165592908859 }, { "label": "completed_group_2", "mean": 0.053928617388010025, "median": 0.052262187004089355, "p05": 0.035437777638435364, "p95": 0.0784272700548172 }, { "label": "completed_group_3", "mean": 0.04293380305171013, "median": 0.03996016085147858, "p05": 0.02556057646870613, "p95": 0.06623870134353638 }, { "label": "completed_group_4", "mean": 0.04174989461898804, "median": 0.03903091698884964, "p05": 0.025013400241732597, "p95": 0.06869364529848099 }, { "label": "completed_group_5", "mean": 0.04734005779027939, "median": 0.04609068110585213, "p05": 0.029884062707424164, "p95": 0.06911227852106094 }, { "label": "completed_group_6", "mean": 0.20163494348526, "median": 0.19222964346408844, "p05": 0.09963390231132507, "p95": 0.3307305574417114 }, { "label": "current_group_partial", "mean": 0.4768889844417572, "median": 0.48383551836013794, "p05": 0.276212215423584, "p95": 0.6487575173377991 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.8757455348968506, "group": 7, "latest_source_mass": 0.3109493851661682, "layer": 28, "max_source_mass": 0.3109493851661682, "mixer_index": 54, "normalized_entropy": 0.9020429270551009, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.11563669890165329, "median": 0.12087491154670715, "p05": 0.05733206868171692, "p95": 0.1688953936100006 }, { "label": "completed_group_1", "mean": 0.08108697831630707, "median": 0.08185897022485733, "p05": 0.05175650864839554, "p95": 0.10753051191568375 }, { "label": "completed_group_2", "mean": 0.07280858606100082, "median": 0.07357072830200195, "p05": 0.04870978742837906, "p95": 0.0948689877986908 }, { "label": "completed_group_3", "mean": 0.07553227245807648, "median": 0.07362619787454605, "p05": 0.056317005306482315, "p95": 0.09756095707416534 }, { "label": "completed_group_4", "mean": 0.07606014609336853, "median": 0.07456053793430328, "p05": 0.05528895929455757, "p95": 0.1022825762629509 }, { "label": "completed_group_5", "mean": 0.06935927271842957, "median": 0.06843917071819305, "p05": 0.04978285729885101, "p95": 0.09167985618114471 }, { "label": "completed_group_6", "mean": 0.19856664538383484, "median": 0.19508783519268036, "p05": 0.1281973123550415, "p95": 0.27796900272369385 }, { "label": "current_group_partial", "mean": 0.3109493851661682, "median": 0.3105033040046692, "p05": 0.18172834813594818, "p95": 0.43916669487953186 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.6924852132797241, "group": 7, "latest_source_mass": 0.3315581679344177, "layer": 28, "max_source_mass": 0.3315581679344177, "mixer_index": 55, "normalized_entropy": 0.8139133413255193, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.08638577908277512, "median": 0.058608733117580414, "p05": 0.026544082909822464, "p95": 0.2740764319896698 }, { "label": "completed_group_1", "mean": 0.059912845492362976, "median": 0.05901291221380234, "p05": 0.03629481792449951, "p95": 0.08707985281944275 }, { "label": "completed_group_2", "mean": 0.06609142571687698, "median": 0.0647939145565033, "p05": 0.038691695779561996, "p95": 0.09703366458415985 }, { "label": "completed_group_3", "mean": 0.056745611131191254, "median": 0.05418796092271805, "p05": 0.032174739986658096, "p95": 0.08370660990476608 }, { "label": "completed_group_4", "mean": 0.06277773529291153, "median": 0.0600711889564991, "p05": 0.03501531109213829, "p95": 0.10094858705997467 }, { "label": "completed_group_5", "mean": 0.05352145433425903, "median": 0.05179818347096443, "p05": 0.032271772623062134, "p95": 0.08084158599376678 }, { "label": "completed_group_6", "mean": 0.2830069661140442, "median": 0.2759752869606018, "p05": 0.12981083989143372, "p95": 0.46204638481140137 }, { "label": "current_group_partial", "mean": 0.3315581679344177, "median": 0.3204721510410309, "p05": 0.13446375727653503, "p95": 0.5667697787284851 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.8980522155761719, "group": 8, "latest_source_mass": 0.20123262703418732, "layer": 29, "max_source_mass": 0.26602888107299805, "mixer_index": 56, "normalized_entropy": 0.9127701729200177, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.11603894829750061, "median": 0.09991352260112762, "p05": 0.030035395175218582, "p95": 0.2941743731498718 }, { "label": "completed_group_1", "mean": 0.08804173022508621, "median": 0.08880250155925751, "p05": 0.05664174258708954, "p95": 0.11779069900512695 }, { "label": "completed_group_2", "mean": 0.07975024729967117, "median": 0.07984748482704163, "p05": 0.05428915470838547, "p95": 0.10499341040849686 }, { "label": "completed_group_3", "mean": 0.09027000516653061, "median": 0.08898856490850449, "p05": 0.06189651042222977, "p95": 0.11996553838253021 }, { "label": "completed_group_4", "mean": 0.08430293202400208, "median": 0.0819794088602066, "p05": 0.05464346706867218, "p95": 0.12159879505634308 }, { "label": "completed_group_5", "mean": 0.07433462142944336, "median": 0.07287181913852692, "p05": 0.052520088851451874, "p95": 0.1010315865278244 }, { "label": "completed_group_6", "mean": 0.26602888107299805, "median": 0.2639757990837097, "p05": 0.15474572777748108, "p95": 0.3846370279788971 }, { "label": "completed_group_7", "mean": 0.20123262703418732, "median": 0.18140269815921783, "p05": 0.08419936150312424, "p95": 0.3782624900341034 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.7512003183364868, "group": 8, "latest_source_mass": 0.33482813835144043, "layer": 29, "max_source_mass": 0.33482813835144043, "mixer_index": 57, "normalized_entropy": 0.7970056117156376, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.07744461297988892, "median": 0.06970229744911194, "p05": 0.014401955530047417, "p95": 0.17144526541233063 }, { "label": "completed_group_1", "mean": 0.046086348593235016, "median": 0.04394831880927086, "p05": 0.028066720813512802, "p95": 0.0718703418970108 }, { "label": "completed_group_2", "mean": 0.05025433003902435, "median": 0.04861236736178398, "p05": 0.029644077643752098, "p95": 0.07761462777853012 }, { "label": "completed_group_3", "mean": 0.03748488053679466, "median": 0.03434944897890091, "p05": 0.020774798467755318, "p95": 0.05938909202814102 }, { "label": "completed_group_4", "mean": 0.04005061462521553, "median": 0.037498839199543, "p05": 0.019541850313544273, "p95": 0.06922487914562225 }, { "label": "completed_group_5", "mean": 0.041623204946517944, "median": 0.039552345871925354, "p05": 0.022081367671489716, "p95": 0.06821337342262268 }, { "label": "completed_group_6", "mean": 0.198423832654953, "median": 0.18318486213684082, "p05": 0.07548617571592331, "p95": 0.3721975088119507 }, { "label": "completed_group_7", "mean": 0.17380405962467194, "median": 0.1514919549226761, "p05": 0.05777408182621002, "p95": 0.36326974630355835 }, { "label": "current_group_partial", "mean": 0.33482813835144043, "median": 0.33602219820022583, "p05": 0.0795319601893425, "p95": 0.5979493856430054 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.0160694122314453, "group": 8, "latest_source_mass": 0.21711516380310059, "layer": 30, "max_source_mass": 0.21711516380310059, "mixer_index": 58, "normalized_entropy": 0.9175527313077866, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.09385271370410919, "median": 0.09211117774248123, "p05": 0.032377734780311584, "p95": 0.16496515274047852 }, { "label": "completed_group_1", "mean": 0.06208505108952522, "median": 0.06140740588307381, "p05": 0.0407925546169281, "p95": 0.0863042101264 }, { "label": "completed_group_2", "mean": 0.06560768187046051, "median": 0.06516212224960327, "p05": 0.041897669434547424, "p95": 0.09085705131292343 }, { "label": "completed_group_3", "mean": 0.07029052078723907, "median": 0.06780634075403214, "p05": 0.04846132919192314, "p95": 0.09917552024126053 }, { "label": "completed_group_4", "mean": 0.06935097277164459, "median": 0.06708167493343353, "p05": 0.04668639227747917, "p95": 0.09815650433301926 }, { "label": "completed_group_5", "mean": 0.056832700967788696, "median": 0.05595892667770386, "p05": 0.038822539150714874, "p95": 0.0771557092666626 }, { "label": "completed_group_6", "mean": 0.19425815343856812, "median": 0.19145764410495758, "p05": 0.12249615788459778, "p95": 0.27496030926704407 }, { "label": "completed_group_7", "mean": 0.17060703039169312, "median": 0.16317546367645264, "p05": 0.09239593893289566, "p95": 0.2726203501224518 }, { "label": "current_group_partial", "mean": 0.21711516380310059, "median": 0.21198657155036926, "p05": 0.11019522696733475, "p95": 0.3391069769859314 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 1.7837426662445068, "group": 8, "latest_source_mass": 0.29026859998703003, "layer": 30, "max_source_mass": 0.29026859998703003, "mixer_index": 59, "normalized_entropy": 0.8118162725118464, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.08836327493190765, "median": 0.07904411852359772, "p05": 0.02631576545536518, "p95": 0.17534467577934265 }, { "label": "completed_group_1", "mean": 0.04953558370471001, "median": 0.04658038914203644, "p05": 0.025181813165545464, "p95": 0.08318150043487549 }, { "label": "completed_group_2", "mean": 0.04813014343380928, "median": 0.045740917325019836, "p05": 0.026714935898780823, "p95": 0.07843323051929474 }, { "label": "completed_group_3", "mean": 0.0373263955116272, "median": 0.03401809185743332, "p05": 0.01990434154868126, "p95": 0.06290259957313538 }, { "label": "completed_group_4", "mean": 0.037015400826931, "median": 0.03405017405748367, "p05": 0.018646355718374252, "p95": 0.06544196605682373 }, { "label": "completed_group_5", "mean": 0.03390160948038101, "median": 0.03198609501123428, "p05": 0.018189728260040283, "p95": 0.05570102855563164 }, { "label": "completed_group_6", "mean": 0.1840541511774063, "median": 0.17327937483787537, "p05": 0.07966600358486176, "p95": 0.32982927560806274 }, { "label": "completed_group_7", "mean": 0.23140482604503632, "median": 0.22406929731369019, "p05": 0.08831030130386353, "p95": 0.40350067615509033 }, { "label": "current_group_partial", "mean": 0.29026859998703003, "median": 0.2745037376880646, "p05": 0.10280418395996094, "p95": 0.5399085879325867 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 1.9973840713500977, "group": 8, "latest_source_mass": 0.19101998209953308, "layer": 31, "max_source_mass": 0.20831164717674255, "mixer_index": 60, "normalized_entropy": 0.9090486661912383, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.10261696577072144, "median": 0.10389091819524765, "p05": 0.01371697522699833, "p95": 0.21163305640220642 }, { "label": "completed_group_1", "mean": 0.06039813905954361, "median": 0.06085003539919853, "p05": 0.026107093319296837, "p95": 0.09348798543214798 }, { "label": "completed_group_2", "mean": 0.06671924144029617, "median": 0.06656201928853989, "p05": 0.034684643149375916, "p95": 0.09819839894771576 }, { "label": "completed_group_3", "mean": 0.08377155661582947, "median": 0.0804532840847969, "p05": 0.05448959022760391, "p95": 0.12039553374052048 }, { "label": "completed_group_4", "mean": 0.07002593576908112, "median": 0.06737451255321503, "p05": 0.044841229915618896, "p95": 0.10655929893255234 }, { "label": "completed_group_5", "mean": 0.05768897384405136, "median": 0.056077782064676285, "p05": 0.039495617151260376, "p95": 0.08242568373680115 }, { "label": "completed_group_6", "mean": 0.20831164717674255, "median": 0.2021285891532898, "p05": 0.11652229726314545, "p95": 0.31706833839416504 }, { "label": "completed_group_7", "mean": 0.1594475358724594, "median": 0.1546279788017273, "p05": 0.06593266129493713, "p95": 0.2735573351383209 }, { "label": "current_group_partial", "mean": 0.19101998209953308, "median": 0.1713225245475769, "p05": 0.06408411264419556, "p95": 0.3850628137588501 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 1.969987154006958, "group": 8, "latest_source_mass": 0.16864418983459473, "layer": 31, "max_source_mass": 0.24503237009048462, "mixer_index": 61, "normalized_entropy": 0.8965797917640489, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.08120635151863098, "median": 0.061164144426584244, "p05": 0.0259944386780262, "p95": 0.23622018098831177 }, { "label": "completed_group_1", "mean": 0.07894632965326309, "median": 0.07658770680427551, "p05": 0.0496862456202507, "p95": 0.11596276611089706 }, { "label": "completed_group_2", "mean": 0.08615730702877045, "median": 0.08467075228691101, "p05": 0.050409119576215744, "p95": 0.12811316549777985 }, { "label": "completed_group_3", "mean": 0.07609779387712479, "median": 0.0722871869802475, "p05": 0.04099677503108978, "p95": 0.11950768530368805 }, { "label": "completed_group_4", "mean": 0.058728985488414764, "median": 0.056080836802721024, "p05": 0.03323862701654434, "p95": 0.09344407916069031 }, { "label": "completed_group_5", "mean": 0.05076971277594566, "median": 0.04946017265319824, "p05": 0.03047419898211956, "p95": 0.07563400268554688 }, { "label": "completed_group_6", "mean": 0.24503237009048462, "median": 0.24110376834869385, "p05": 0.10613486170768738, "p95": 0.3991503119468689 }, { "label": "completed_group_7", "mean": 0.1544169783592224, "median": 0.1400495171546936, "p05": 0.06199025735259056, "p95": 0.2995939254760742 }, { "label": "current_group_partial", "mean": 0.16864418983459473, "median": 0.15222924947738647, "p05": 0.0543956533074379, "p95": 0.3329208791255951 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.071648359298706, "group": 8, "latest_source_mass": 0.11520501971244812, "layer": 32, "max_source_mass": 0.20955611765384674, "mixer_index": 62, "normalized_entropy": 0.9428478002054053, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.10569609701633453, "median": 0.08689188957214355, "p05": 0.03173588588833809, "p95": 0.24571751058101654 }, { "label": "completed_group_1", "mean": 0.09922641515731812, "median": 0.09795533865690231, "p05": 0.06407676637172699, "p95": 0.14120742678642273 }, { "label": "completed_group_2", "mean": 0.08180510997772217, "median": 0.080217644572258, "p05": 0.054607540369033813, "p95": 0.11442367732524872 }, { "label": "completed_group_3", "mean": 0.10646799206733704, "median": 0.10510502755641937, "p05": 0.07415887713432312, "p95": 0.14216329157352448 }, { "label": "completed_group_4", "mean": 0.0826798751950264, "median": 0.08022409677505493, "p05": 0.05491747707128525, "p95": 0.11860857903957367 }, { "label": "completed_group_5", "mean": 0.06615597754716873, "median": 0.06485182791948318, "p05": 0.04477996751666069, "p95": 0.09238749742507935 }, { "label": "completed_group_6", "mean": 0.20955611765384674, "median": 0.206287682056427, "p05": 0.129981130361557, "p95": 0.2996031939983368 }, { "label": "completed_group_7", "mean": 0.13320739567279816, "median": 0.12465103715658188, "p05": 0.06303662061691284, "p95": 0.23065781593322754 }, { "label": "current_group_partial", "mean": 0.11520501971244812, "median": 0.10386130958795547, "p05": 0.03912060335278511, "p95": 0.22735844552516937 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 1.9781701564788818, "group": 8, "latest_source_mass": 0.16112206876277924, "layer": 32, "max_source_mass": 0.22251944243907928, "mixer_index": 63, "normalized_entropy": 0.9003040366848136, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.09446848928928375, "median": 0.074150450527668, "p05": 0.042592622339725494, "p95": 0.19443637132644653 }, { "label": "completed_group_1", "mean": 0.07056106626987457, "median": 0.06868506222963333, "p05": 0.04818308353424072, "p95": 0.0982733890414238 }, { "label": "completed_group_2", "mean": 0.08033448457717896, "median": 0.08047192543745041, "p05": 0.048296794295310974, "p95": 0.11253741383552551 }, { "label": "completed_group_3", "mean": 0.0658978745341301, "median": 0.06256154924631119, "p05": 0.04223886504769325, "p95": 0.0967884361743927 }, { "label": "completed_group_4", "mean": 0.058234501630067825, "median": 0.0563921295106411, "p05": 0.03529864549636841, "p95": 0.08846980333328247 }, { "label": "completed_group_5", "mean": 0.04316519945859909, "median": 0.042268507182598114, "p05": 0.029443470761179924, "p95": 0.05996086448431015 }, { "label": "completed_group_6", "mean": 0.2036968618631363, "median": 0.19645483791828156, "p05": 0.09398645162582397, "p95": 0.3395216464996338 }, { "label": "completed_group_7", "mean": 0.22251944243907928, "median": 0.2130982130765915, "p05": 0.11867167800664902, "p95": 0.35465875267982483 }, { "label": "current_group_partial", "mean": 0.16112206876277924, "median": 0.14023703336715698, "p05": 0.06008770316839218, "p95": 0.3247341811656952 } ], "sources": 9 }, { "branch": "output", "entropy_mean": 1.6036605834960938, "group": null, "latest_source_mass": 0.31639498472213745, "layer": null, "max_source_mass": 0.3412485122680664, "mixer_index": null, "normalized_entropy": 0.7298573846467136, "offset": null, "source_summaries": [ { "label": "embedding", "mean": 0.09759747982025146, "median": 0.09915056824684143, "p05": 0.025461740791797638, "p95": 0.16398483514785767 }, { "label": "completed_group_1", "mean": 0.04215940088033676, "median": 0.04050712287425995, "p05": 0.025720715522766113, "p95": 0.06398092210292816 }, { "label": "completed_group_2", "mean": 0.041145481169223785, "median": 0.03881259262561798, "p05": 0.023510465398430824, "p95": 0.0662468671798706 }, { "label": "completed_group_3", "mean": 0.029554028064012527, "median": 0.027420252561569214, "p05": 0.01627083122730255, "p95": 0.04924973472952843 }, { "label": "completed_group_4", "mean": 0.0289996936917305, "median": 0.027226522564888, "p05": 0.01560397818684578, "p95": 0.048351090401411057 }, { "label": "completed_group_5", "mean": 0.03647579252719879, "median": 0.0347352996468544, "p05": 0.020744025707244873, "p95": 0.058257561177015305 }, { "label": "completed_group_6", "mean": 0.06642461568117142, "median": 0.057809896767139435, "p05": 0.022837381809949875, "p95": 0.14078694581985474 }, { "label": "completed_group_7", "mean": 0.3412485122680664, "median": 0.3336990475654602, "p05": 0.0819513276219368, "p95": 0.6423859596252441 }, { "label": "completed_group_8", "mean": 0.31639498472213745, "median": 0.30713334679603577, "p05": 0.05841828137636185, "p95": 0.6156357526779175 } ], "sources": 9 } ], "mode": "uniform_group_7_only", "optimizer_state_after": "ed03e6fbd4a12d8b063dcb22e0437754285f54d585374cd52fbd534f05d24637", "optimizer_state_before": "ed03e6fbd4a12d8b063dcb22e0437754285f54d585374cd52fbd534f05d24637", "optimizer_state_unchanged": true, "positions": { "attention_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "5e6ecbcd78543fc42d4098a51169dca446d404ce384015643e84d2c78433c949", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.0933690071105957, 0.24537226557731628, 0.07698935270309448, 0.0523640513420105, 0.09380833059549332, 0.03647654876112938, 0.039120711386203766, 0.06371714919805527, 0.05156813934445381, 0.03728010877966881, 0.03983496502041817, 0.040284499526023865, 0.046871740370988846, 0.04314620420336723, 0.04033593460917473, 0.04471700266003609, 0.08397205173969269, 0.0528092160820961, 0.04443390294909477, 0.07560808956623077, 0.03708760067820549, 0.03599191829562187, 0.033304885029792786, 0.04061548039317131, 0.025487789884209633, 0.021565141156315804, 0.02342291921377182, 0.017244853079319, 0.024166326969861984, 0.016819356009364128, 0.021590016782283783, 0.026201345026493073 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.4685368228128937e-05, "first_to_last_ratio": 0.9410133690271989, "imbalance_abs_log_ratio": 0.06079793224242032, "last_quartile_mean": 1.560590817462071e-05, "mean": 1.5210201098625475e-05, "normalized": [ 0.8983760459169909, 1.0440106519271874, 0.9321244306915835, 0.8651462037969389, 1.0983308651525907, 1.0263697973344956, 0.9818187196338675, 0.8777807180123939, 0.8391239371488647, 0.7681085878068088, 0.7294759043475437, 0.7043343589585478, 0.8394131657950837, 0.7867050868534748, 0.7506006576053812, 0.7129969871237437, 0.7986287575406578, 0.7565233570901861, 0.7558443245273446, 0.7482836592980949, 2.231812847106165, 1.9001614083228633, 1.529772704541691, 1.216129620714873, 1.2049901620796157, 1.1804635489622173, 1.1440033989747485, 1.0871965488914441, 1.3764448078862193, 0.9822149216206046, 0.6495913349501353, 0.5832224793876426 ], "peak_layer": 21, "peak_normalized": 2.231812847106165, "population_cv": 0.350716819764397, "rest_mean": 1.3473496538709798e-05, "spike_contrast": 1.8249461564432692, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.4588405722170137e-05, "top_five_layers": [ 21, 22, 23, 29, 24 ], "top_five_spike_overlap": 4 }, "values": [ 1.3664480320585426e-05, 1.5879611964919604e-05, 1.4177800039760768e-05, 1.3159047739463858e-05, 1.6705833331798203e-05, 1.561129101901315e-05, 1.4933660168026108e-05, 1.3351221241464373e-05, 1.2763243830704596e-05, 1.1683086086122785e-05, 1.1095475201727822e-05, 1.0713067240430973e-05, 1.2767643056577072e-05, 1.1965942576352973e-05, 1.1416786946938373e-05, 1.0844827556866221e-05, 1.214730400533881e-05, 1.1506872397148982e-05, 1.1496544175315648e-05, 1.1381544936739374e-05, 3.394632221898064e-05, 2.8901837140438147e-05, 2.326815047126729e-05, 1.8497576093068346e-05, 1.832814268709626e-05, 1.7955087969312444e-05, 1.7400521755916998e-05, 1.653647814237047e-05, 2.0936002329108305e-05, 1.4939686479920056e-05, 9.880414836516138e-06, 8.870931196724996e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 5.8246871958544943e-05, "first_to_last_ratio": 0.935035547727828, "imbalance_abs_log_ratio": 0.0671707314559085, "last_quartile_mean": 6.229375139810145e-05, "mean": 6.064571823571896e-05, "normalized": [ 0.8976470372611476, 1.0254318915001792, 0.9269495612220872, 0.8659737513632113, 1.0947251339860233, 1.0237809183459061, 0.9756001204389475, 0.8734509405132554, 0.8378731348987841, 0.766914342321072, 0.7313053432598954, 0.7062387093993366, 0.8453415059013942, 0.7914586659115103, 0.7547553167575569, 0.7152089848734072, 0.8029982016209386, 0.7592988222323696, 0.7575945802691075, 0.7514147385650144, 2.2289822028807937, 1.9091292793821353, 1.5302433578677603, 1.2102853364486639, 1.1992850874428513, 1.1752432182939576, 1.1403556276752376, 1.0851108338122843, 1.3786969555883997, 0.9878922580375376, 0.6581690681399641, 0.5926450737892698 ], "peak_layer": 21, "peak_normalized": 2.2289822028807937, "population_cv": 0.3496267030085714, "rest_mean": 5.37322742241452e-05, "spike_contrast": 1.823453730796855, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.797831589821726e-05, "top_five_layers": [ 21, 22, 23, 29, 24 ], "top_five_spike_overlap": 4 }, "values": [ 5.4438449296867475e-05, 6.21880535618402e-05, 5.621552190859802e-05, 5.251760012470186e-05, 6.639039202127606e-05, 6.208792910911143e-05, 5.916597001487389e-05, 5.297105963109061e-05, 5.08134180563502e-05, 4.651007111533545e-05, 4.435053779161535e-05, 4.2830353777389973e-05, 5.126634277985431e-05, 4.799857924808748e-05, 4.577267827698961e-05, 4.337436257628724e-05, 4.8698402679292485e-05, 4.604822242981754e-05, 4.594486745190807e-05, 4.5570086513180286e-05, 0.00013517822662834078, 0.00011578051635297015, 9.280270751332864e-05, 7.339862349908799e-05, 7.273150549735874e-05, 7.127346907509491e-05, 6.91576860845089e-05, 6.580732588190585e-05, 8.361206710105762e-05, 5.991143552819267e-05, 3.991513585788198e-05, 3.594138615881093e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.05165390809997916, "first_to_last_ratio": 0.9350355438255078, "imbalance_abs_log_ratio": 0.06717073562935438, "last_quartile_mean": 0.05524272145703435, "mean": 0.05378122895490378, "normalized": [ 0.8976470635169106, 1.0254319910336536, 0.9269495634177769, 0.8659737455160643, 1.0947251018139446, 1.0237808620697004, 0.9756000641408223, 0.8734509623752265, 0.8378731056921183, 0.7669143197766936, 0.73130529272677, 0.7062387071411542, 0.8453415255810903, 0.7914586978169286, 0.7547553138218569, 0.7152089845707745, 0.8029981754144385, 0.7592988450238346, 0.7575946571688135, 0.7514147509765086, 2.2289821252427546, 1.9091292763884413, 1.5302433546947969, 1.210285357803093, 1.1992851275514895, 1.175243215824319, 1.1403556095974676, 1.0851109138733996, 1.3786969102172586, 0.9878922021957017, 0.6581690835205845, 0.5926450934956127 ], "peak_layer": 21, "peak_normalized": 2.2289821252427546, "population_cv": 0.34962669180878264, "rest_mean": 0.0476503177649445, "spike_contrast": 1.823453724050629, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0868881493806839, "top_five_layers": [ 21, 22, 23, 29, 24 ], "top_five_spike_overlap": 4 }, "values": [ 0.04827656224370003, 0.05514899268746376, 0.049852486699819565, 0.046573132276535034, 0.058875661343336105, 0.055060192942619324, 0.05246897041797638, 0.0469752661883831, 0.04506184533238411, 0.04124559462070465, 0.03933049738407135, 0.037982385605573654, 0.04546350613236427, 0.04256562143564224, 0.04059166833758354, 0.03846481814980507, 0.04318622872233391, 0.04083602502942085, 0.040744371712207794, 0.04041200876235962, 0.1198773980140686, 0.10267531871795654, 0.08229836821556091, 0.06509063392877579, 0.06449902802705765, 0.0632060244679451, 0.06132972612977028, 0.05835859850049019, 0.0741480141878128, 0.05313005670905113, 0.03539714217185974, 0.0318731814622879 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 2.452364981309074e-06, "first_to_last_ratio": 0.5838820251514151, "imbalance_abs_log_ratio": 0.5380563282880563, "last_quartile_mean": 4.2001035751582094e-06, "mean": 3.366456766684678e-06, "normalized": [ 0.8053237293484851, 0.7248080322732592, 0.6875597407584304, 0.6713851700386589, 0.7472411496929621, 0.7349727420570951, 0.7286710366925266, 0.7278036758611212, 0.6791759506446224, 0.6550753823445477, 0.6522043518024684, 0.6593189124548396, 0.6955335573286602, 0.697894045858234, 0.7093304841185168, 0.7222060848609354, 0.7203620144061883, 0.7486009515632848, 0.7725028819405155, 0.7997177632709676, 2.2366963764514525, 2.100675259605763, 1.810669565692931, 1.5312047170934548, 1.4728383979228457, 1.4587371991702511, 1.4264241245959917, 1.3612988478548795, 1.6176883321770112, 1.1674416082664376, 0.7872779367135846, 0.6893599771390783 ], "peak_layer": 21, "peak_normalized": 2.2366963764514525, "population_cv": 0.45472771814262447, "rest_mean": 2.8487600132023175e-06, "spike_contrast": 2.1630531202811434, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.162019235489424e-06, "top_five_layers": [ 21, 22, 23, 29, 24 ], "top_five_spike_overlap": 4 }, "values": [ 2.7110875180369476e-06, 2.4400349047937198e-06, 2.314640141776181e-06, 2.2601891487283865e-06, 2.5155550247291103e-06, 2.4742539608269e-06, 2.4530395421606954e-06, 2.4501196094206534e-06, 2.286416474817088e-06, 2.205282953582355e-06, 2.195617753386614e-06, 2.2195686142367776e-06, 2.3414836505253334e-06, 2.349430133108399e-06, 2.3879304080764996e-06, 2.4312755613209447e-06, 2.425067577860318e-06, 2.520132738936809e-06, 2.6005975541920634e-06, 2.692215275601484e-06, 7.529741651524091e-06, 7.0718324423069134e-06, 6.095540811656974e-06, 5.1547344810387585e-06, 4.958246790920384e-06, 4.910775714961346e-06, 4.801995146408444e-06, 4.582753717841115e-06, 5.44587783224415e-06, 3.930141701857792e-06, 2.6503371373109985e-06, 2.3207005597214447e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 4.4230577259440906e-05, "first_to_last_ratio": 1.1345895733514464, "imbalance_abs_log_ratio": 0.1262709761549421, "last_quartile_mean": 3.898376849065244e-05, "mean": 4.2956183449405216e-05, "normalized": [ 0.9773979974208342, 1.1018689171137157, 1.0009097620917895, 0.9371393724027711, 1.178698405890451, 1.0987482431543636, 1.0290742510891369, 0.9135014189508932, 0.8860175024659579, 0.8078494810164164, 0.7639912575046943, 0.7314445455005971, 0.9018818882511775, 0.8345003821663488, 0.7871031138377192, 0.7368849757674997, 0.861818051284203, 0.8084737343749266, 0.8061274699543368, 0.8009727005988784, 2.283256075632844, 1.9041077400446695, 1.4813556757680426, 1.1066848399507372, 1.128554541351016, 1.0957905136342696, 1.060839439556084, 1.01037595323195, 1.1008140974718428, 0.7825987671919825, 0.5651122338910219, 0.5161066514388285 ], "peak_layer": 21, "peak_normalized": 2.283256075632844, "population_cv": 0.34207980345839, "rest_mean": 3.833607270617331e-05, "spike_contrast": 1.7713025009972578, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.79047814628575e-05, "top_five_layers": [ 21, 22, 23, 5, 25 ], "top_five_spike_overlap": 4 }, "values": [ 4.198528768029064e-05, 4.7332083340734243e-05, 4.299526335671544e-05, 4.025593079859391e-05, 5.0632384954951704e-05, 4.719803109765053e-05, 4.420510231284425e-05, 3.924053453374654e-05, 3.8059930375311524e-05, 3.470213050604798e-05, 3.281814861111343e-05, 3.142006607959047e-05, 3.874140384141356e-05, 3.584695150493644e-05, 3.381094575161114e-05, 3.1653766200179234e-05, 3.702041431097314e-05, 3.472894604783505e-05, 3.462815948296338e-05, 3.440673026489094e-05, 9.80799668468535e-05, 8.17932013887912e-05, 6.363338616210967e-05, 4.7538957005599514e-05, 4.8478395910933614e-05, 4.707097832579166e-05, 4.5569613575935364e-05, 4.340189479989931e-05, 4.7286772314691916e-05, 3.3617456210777164e-05, 2.4275064788525924e-05, 2.216997199866455e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 3.1976308946468635e-05, "first_to_last_ratio": 1.3333074193160395, "imbalance_abs_log_ratio": 0.28766263674993847, "last_quartile_mean": 2.3982697825886135e-05, "mean": 2.9701665454240356e-05, "normalized": [ 1.0360145055849732, 1.1578192098431006, 1.05436395618763, 0.9866207501452423, 1.2435903459867665, 1.1622797096902209, 1.0492343262456414, 0.9227414243834177, 0.9220689872205352, 0.8307217057109665, 0.7792206742842025, 0.7398611756194843, 0.9434744748782918, 0.8639334261793317, 0.8040507533546389, 0.7470264896501991, 0.9171919192876291, 0.8499194805012595, 0.8540364739689074, 0.8538711205681986, 2.414044270420027, 1.9626330385210105, 1.4506015351461394, 0.9950565294714031, 1.0696689444732361, 1.0361193518894227, 1.007227458393575, 0.9702358204270068, 0.8554973406431695, 0.6052780830385812, 0.47189943687684116, 0.44369728140895 ], "peak_layer": 21, "peak_normalized": 2.414044270420027, "population_cv": 0.36963171039013454, "rest_mean": 2.6520282315484816e-05, "spike_contrast": 1.767746430668539, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.688113440352026e-05, "top_five_layers": [ 21, 22, 23, 5, 6 ], "top_five_spike_overlap": 3 }, "values": [ 3.0771356250625104e-05, 3.4389158827252686e-05, 3.131636549369432e-05, 2.930427945102565e-05, 3.693670441862196e-05, 3.4521643101470545e-05, 3.116400694125332e-05, 2.7406957087805495e-05, 2.738698458415456e-05, 2.4673818188603036e-05, 2.3144151782616973e-05, 2.1975109120830894e-05, 2.802276321745012e-05, 2.5660261599114165e-05, 2.3881646484369412e-05, 2.218793088104576e-05, 2.7242127544013783e-05, 2.524402407289017e-05, 2.536630563554354e-05, 2.5361394364153966e-05, 7.170113531174138e-05, 5.829346991959028e-05, 4.308528150431812e-05, 2.9554836146417074e-05, 3.1770949135534465e-05, 3.0774470360483974e-05, 2.991633300553076e-05, 2.8817619750043377e-05, 2.5409695808775723e-05, 1.7977767129195854e-05, 1.4016199202160351e-05, 1.3178548215364572e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00012399425395415165, "first_to_last_ratio": 0.9334705715201977, "imbalance_abs_log_ratio": 0.06884584138190952, "last_quartile_mean": 0.000132831454720872, "mean": 0.00012800334957319137, "normalized": [ 0.9125598064823551, 1.0019872845132374, 0.9286772792861776, 0.8788460043457256, 1.1047185029021847, 1.034897739633526, 1.002870380759319, 0.8848810827272406, 0.8586568303125444, 0.7924252371176711, 0.7563175938824653, 0.7366724539746219, 0.8611628773836852, 0.8138526557842611, 0.7779340119709285, 0.7320866743177683, 0.8100293537950787, 0.770008370951368, 0.7756001959519007, 0.760406916110555, 2.062909875066142, 1.8046546987223542, 1.4493688676049015, 1.1867266309964009, 1.2661994952667412, 1.2451906037409788, 1.201236898190159, 1.1412766184436596, 1.3512708350435871, 0.9238031973533117, 0.6138340844594897, 0.5589369429096605 ], "peak_layer": 21, "peak_normalized": 2.062909875066142, "population_cv": 0.32087523251142047, "rest_mean": 0.00011487181985069549, "spike_contrast": 1.7316136397352009, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001989136100746691, "top_five_layers": [ 21, 22, 23, 29, 25 ], "top_five_spike_overlap": 4 }, "values": [ 0.00011681071191560477, 0.00012825772864744067, 0.00011887380242114887, 0.00011249523231526837, 0.00014140766870696098, 0.00013247037713881582, 0.00012837076792493463, 0.00011326774256303906, 0.00010991095041390508, 0.00010143308463739231, 9.681118535809219e-05, 9.429654164705426e-05, 0.0001102317328331992, 0.00010417586599942297, 9.957815927919e-05, 9.370954649057239e-05, 0.00010368647053837776, 9.856365068117157e-05, 9.927942301146686e-05, 9.733463230077177e-05, 0.0002640593738760799, 0.00023100184625945985, 0.0001855240698205307, 0.00015190498379524797, 0.00016207777662202716, 0.00015938856813590974, 0.00015376234659925103, 0.0001460872299503535, 0.0001729671930661425, 0.00011824990360764787, 7.857281889300793e-05, 7.154580089263618e-05 ] } } }, "mlp_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "c0b8018c349b1bd09e5d17858eb312a2323da0e92de9651e2bb08ae4adf115c7", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.13583384454250336, 0.0415157787501812, 0.018140379339456558, 0.039518751204013824, 0.022877270355820656, 0.015532293356955051, 0.02314678393304348, 0.05432315170764923, 0.03743875026702881, 0.04006272554397583, 0.045583780854940414, 0.05995568633079529, 0.038651175796985626, 0.04559725895524025, 0.05632574483752251, 0.07952121645212173, 0.042337141931056976, 0.05044656619429588, 0.04418695345520973, 0.06310147047042847, 0.029010163620114326, 0.04586305841803551, 0.06033914536237717, 0.07656749337911606, 0.045884523540735245, 0.051604680716991425, 0.06502768397331238, 0.06792232394218445, 0.060306549072265625, 0.08025261014699936, 0.09663844108581543, 0.12230555713176727 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.4557177109963959e-05, "first_to_last_ratio": 1.0607619949982274, "imbalance_abs_log_ratio": 0.05898751307145722, "last_quartile_mean": 1.3723320762437652e-05, "mean": 1.4120558176955456e-05, "normalized": [ 0.9661131994393504, 1.1234771947632944, 1.004887380680764, 0.9301102910150655, 1.1780742192048355, 1.0933683133046266, 1.0326639384286282, 0.9186718505466533, 0.8478387330933669, 0.8013733777161709, 0.7660210028170336, 0.7425279184905968, 0.8825265976579549, 0.8223703455838384, 0.7772059928476778, 0.7449630393802261, 0.8551725002458773, 0.8025248864649421, 0.7989177746818885, 0.8260363918397147, 2.177547228678574, 1.6956099652823422, 1.3567608356145802, 1.0802918165380408, 1.2722332145458177, 1.2327704339066463, 1.1723348388722756, 1.118675742073626, 1.0579235755267702, 0.7039689285865611, 0.6306446541875188, 0.5863938179847412 ], "peak_layer": 21, "peak_normalized": 2.177547228678574, "population_cv": 0.3121765028922915, "rest_mean": 1.2769982714854888e-05, "spike_contrast": 1.6768750710514846, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.141366567229852e-05, "top_five_layers": [ 21, 22, 23, 25, 26 ], "top_five_spike_overlap": 4 }, "values": [ 1.3642057638207916e-05, 1.5864125089137815e-05, 1.4189570720191114e-05, 1.31336764752632e-05, 1.6635065549053252e-05, 1.543897087685764e-05, 1.458179121982539e-05, 1.2972159311175346e-05, 1.1971956155321095e-05, 1.131583940150449e-05, 1.0816644135047682e-05, 1.0484908671060111e-05, 1.2461768164939713e-05, 1.1612328307819553e-05, 1.097458243748406e-05, 1.051929393725004e-05, 1.2075513041054364e-05, 1.1332099347782787e-05, 1.1281164915999398e-05, 1.1664094927255064e-05, 3.074818232562393e-05, 2.3942959160194732e-05, 1.9158220311510377e-05, 1.5254323443514295e-05, 1.796464312064927e-05, 1.740740663080942e-05, 1.655402229516767e-05, 1.579632589709945e-05, 1.4938471394998487e-05, 9.940434210875537e-06, 8.905054528440814e-06, 8.280208021460567e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 5.775077397629502e-05, "first_to_last_ratio": 1.0524251097415505, "imbalance_abs_log_ratio": 0.0510971294029309, "last_quartile_mean": 5.4873998578841565e-05, "mean": 5.63304391789643e-05, "normalized": [ 0.9642729617008539, 1.1028115137200063, 0.998794999552996, 0.9304682452685216, 1.1736725625786015, 1.0906908677504163, 1.0265032266194138, 0.9145003359245287, 0.8458374926269373, 0.7999924066745355, 0.7680416047715146, 0.7450038137469073, 0.8894201131090249, 0.8283290891957269, 0.7813705953674341, 0.7472622750821689, 0.859062892374079, 0.8034806541335047, 0.7989585648371071, 0.8287473275602438, 2.1774264250628357, 1.707350398990564, 1.3491332439169343, 1.0757108172735714, 1.2661018203147099, 1.2278323641331839, 1.1682881634237994, 1.1168082907101666, 1.0638830544082254, 0.712758707917527, 0.6406727166469813, 0.5968124546069786 ], "peak_layer": 21, "peak_normalized": 2.1774264250628357, "population_cv": 0.3104700535978533, "rest_mean": 5.0956676541655154e-05, "spike_contrast": 1.674927864470602, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 8.534875742043369e-05, "top_five_layers": [ 21, 22, 23, 25, 26 ], "top_five_spike_overlap": 4 }, "values": [ 5.431791942100972e-05, 6.212185689946637e-05, 5.6262560974573717e-05, 5.241368489805609e-05, 6.611349090235308e-05, 6.143909558886662e-05, 5.78233775740955e-05, 5.1514205551939085e-05, 4.764639743370935e-05, 4.5063923607813194e-05, 4.326412090449594e-05, 4.1966392018366605e-05, 5.0101425586035475e-05, 4.666014137910679e-05, 4.401494879857637e-05, 4.20936121372506e-05, 4.839139000978321e-05, 4.526041811914183e-05, 4.500568684306927e-05, 4.6683700929861516e-05, 0.00012265538680367172, 9.617579780751839e-05, 7.599726814078167e-05, 6.059526276658289e-05, 7.132007158361375e-05, 6.916433630976826e-05, 6.581018533324823e-05, 6.291030149441212e-05, 5.992899968987331e-05, 4.015001104562543e-05, 3.608937549870461e-05, 3.36187076754868e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.05121396388858557, "first_to_last_ratio": 1.052425086966144, "imbalance_abs_log_ratio": 0.05109710776204978, "last_quartile_mean": 0.0486628117505461, "mean": 0.049954396847169846, "normalized": [ 0.9642730020707997, 1.1028114758527232, 0.9987950140450647, 0.9304682411165044, 1.1736725655480387, 1.0906908434158626, 1.026503215903439, 0.9145003452028047, 0.8458374680585027, 0.7999924652474599, 0.7680415349341949, 0.7450038169736878, 0.8894201248316143, 0.8283291007038053, 0.7813706361565212, 0.7472622851728751, 0.8590629116682529, 0.8034806557713823, 0.7989585737792898, 0.8287473106975659, 2.17742629131299, 1.7073503535008319, 1.3491332097652093, 1.0757108269221607, 1.2661018214675832, 1.2278324003641332, 1.1682882610260137, 1.1168083110772131, 1.0638830458842834, 0.7127586929326434, 0.6406727282110227, 0.5968124703855264 ], "peak_layer": 21, "peak_normalized": 2.17742629131299, "population_cv": 0.3104700337371385, "rest_mean": 0.045188890770077705, "spike_contrast": 1.6749278057868395, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.07568812966346741, "top_five_layers": [ 21, 22, 23, 25, 26 ], "top_five_spike_overlap": 4 }, "values": [ 0.04816967621445656, 0.05509028211236, 0.04989420250058174, 0.04648097977042198, 0.05863010510802269, 0.05448480322957039, 0.05127834901213646, 0.04568331316113472, 0.04225330054759979, 0.039963141083717346, 0.038367051631212234, 0.03721621632575989, 0.0444304458796978, 0.0413786806166172, 0.03903289884328842, 0.03732903674244881, 0.04291396960616112, 0.0401373915374279, 0.03991149365901947, 0.04139957204461098, 0.10877201706171036, 0.08528965711593628, 0.06739513576030731, 0.05373648554086685, 0.06324735283851624, 0.06133562698960304, 0.05836113542318344, 0.055789485573768616, 0.0531456358730793, 0.035605430603027344, 0.03200441971421242, 0.029813406988978386 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 2.4365399156067724e-06, "first_to_last_ratio": 0.6571209831462147, "imbalance_abs_log_ratio": 0.41988713263090216, "last_quartile_mean": 3.7079015555718797e-06, "mean": 3.086074130465022e-06, "normalized": [ 0.8749224644045405, 0.7901821248586449, 0.751646307082443, 0.7342562489224403, 0.8141199602450068, 0.795372618470957, 0.7852838355028365, 0.7704352028657384, 0.7052645913548845, 0.6974222299149532, 0.6980907042516294, 0.7114438346198139, 0.7418866407088263, 0.7402650028657485, 0.7479899963327574, 0.7670700648900575, 0.7813774639244637, 0.7998448318773551, 0.821356325716773, 0.866563772296536, 2.1094511145009536, 1.8446564424432927, 1.613709868590465, 1.42543115772753, 1.5802827625998683, 1.5432136392599296, 1.4958606354049158, 1.4164677221658235, 1.2619208466946197, 0.853428137380999, 0.7627057887594481, 0.6980776633657492 ], "peak_layer": 21, "peak_normalized": 2.1094511145009536, "population_cv": 0.39131583064826364, "rest_mean": 2.677622921543877e-06, "spike_contrast": 1.976271795428194, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.291710658639204e-06, "top_five_layers": [ 21, 22, 23, 25, 26 ], "top_five_spike_overlap": 4 }, "values": [ 2.7000755835615564e-06, 2.438560613882146e-06, 2.319636223546695e-06, 2.265969214931829e-06, 2.5124345484073274e-06, 2.454578861943446e-06, 2.4234441298176534e-06, 2.3776201487635262e-06, 2.1764988105132943e-06, 2.152296701751766e-06, 2.1543596631090622e-06, 2.1955684132990427e-06, 2.2895171696291072e-06, 2.2845126750326017e-06, 2.308352577529149e-06, 2.367235083511332e-06, 2.4113887775456533e-06, 2.4683804440428503e-06, 2.5347665086883353e-06, 2.6742800400825217e-06, 6.509922513942001e-06, 5.692746526619885e-06, 4.980028279533144e-06, 4.398986220621737e-06, 4.8768697524792515e-06, 4.762471689900849e-06, 4.61633680970408e-06, 4.371324394014664e-06, 3.8943812796787824e-06, 2.6337424969824497e-06, 2.353766603846452e-06, 2.1543194179685088e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 4.3904467929678503e-05, "first_to_last_ratio": 1.2504655848875463, "imbalance_abs_log_ratio": 0.22351594987529447, "last_quartile_mean": 3.511049681037548e-05, "mean": 4.036382063077326e-05, "normalized": [ 1.0382198995599758, 1.1714354630507813, 1.066167224983976, 0.9959159954184936, 1.2500606424972418, 1.15797558466013, 1.071732643466384, 0.9502392578635362, 0.8869454097764992, 0.8336861511846586, 0.7927071535467861, 0.7629611999468652, 0.9387987352697132, 0.8642348899701341, 0.8052963790922082, 0.7607393227273422, 0.9118893434032349, 0.8474217363096325, 0.8424659550782733, 0.8780427591097948, 2.2287451022151146, 1.7083158790373993, 1.2963551289757191, 0.9808427053733445, 1.1774665816627234, 1.1341270873357665, 1.079429268696027, 1.0321078444827112, 0.8401073511127656, 0.6107082171391409, 0.5549582695150553, 0.5299008175385715 ], "peak_layer": 21, "peak_normalized": 2.2287451022151146, "population_cv": 0.31675765638367587, "rest_mean": 3.678829563324581e-05, "spike_contrast": 1.6220282725872153, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.9671655617421496e-05, "top_five_layers": [ 21, 22, 23, 5, 25 ], "top_five_spike_overlap": 4 }, "values": [ 4.19065218011383e-05, 4.7283610911108553e-05, 4.303458263166249e-05, 4.019897460239008e-05, 5.045722355134785e-05, 4.6740318794036284e-05, 4.3259224185021594e-05, 3.835528696072288e-05, 3.58005054295063e-05, 3.365075826877728e-05, 3.199668935849331e-05, 3.07960290228948e-05, 3.78935037588235e-05, 3.488382208161056e-05, 3.2504838600289077e-05, 3.0706345569342375e-05, 3.6807337892241776e-05, 3.4205178963020444e-05, 3.4005144698312506e-05, 3.544116043485701e-05, 8.99606675375253e-05, 6.895415572216734e-05, 5.232584589975886e-05, 3.959055902669206e-05, 4.75270499009639e-05, 4.57777023257222e-05, 4.356988938525319e-05, 4.165981590631418e-05, 3.390994243090972e-05, 2.4650516934343614e-05, 2.2400236048270017e-05, 2.138882155122701e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 3.1791102401257376e-05, "first_to_last_ratio": 1.4439952817842054, "imbalance_abs_log_ratio": 0.36741377300283473, "last_quartile_mean": 2.2016070829522505e-05, "mean": 2.8182190561665266e-05, "normalized": [ 1.0900372941800265, 1.2199863042011063, 1.1128933333589761, 1.0393257073594704, 1.311421433385419, 1.2114384930527116, 1.0851838486364749, 0.9541653456549098, 0.9148784866832742, 0.8482790881736264, 0.8042050702770522, 0.7603290731651956, 0.9730230173072366, 0.8841567366339266, 0.8179955914255278, 0.766941469955527, 0.9607599928638211, 0.8824791107420176, 0.885405016287263, 0.9341649236787816, 2.366682198592525, 1.7613632535351451, 1.2809103547315466, 0.8843341033602641, 1.1025910890390718, 1.0643111799573897, 1.0293906534712016, 0.988948587196804, 0.6399084902580298, 0.5094652028023454, 0.46882960154653475, 0.4461959484867977 ], "peak_layer": 21, "peak_normalized": 2.366682198592525, "population_cv": 0.3534141269738911, "rest_mean": 2.5681406306695727e-05, "spike_contrast": 1.6232142835430383, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.168642553850077e-05, "top_five_layers": [ 21, 22, 5, 23, 2 ], "top_five_spike_overlap": 3 }, "values": [ 3.071963874390349e-05, 3.438188650761731e-05, 3.136377199552953e-05, 2.929047514044214e-05, 3.695872874232009e-05, 3.414099046494812e-05, 3.0582858016714454e-05, 2.6890469598583877e-05, 2.578327985247597e-05, 2.3906362912384793e-05, 2.2664260541205294e-05, 2.1427738829515874e-05, 2.7421920094639063e-05, 2.4917473638197407e-05, 2.3052907636156306e-05, 2.161409065593034e-05, 2.7076321202912368e-05, 2.4870194465620443e-05, 2.4952652893261984e-05, 2.6326813895138912e-05, 6.669828871963546e-05, 4.963907485944219e-05, 3.60988597094547e-05, 2.492247222107835e-05, 3.107343218289316e-05, 2.999462049046997e-05, 2.9010483558522537e-05, 2.7870737540069968e-05, 1.8034023014479317e-05, 1.435784542991314e-05, 1.3212645171734039e-05, 1.2574779248097911e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00012292057544982526, "first_to_last_ratio": 1.0512587147591843, "imbalance_abs_log_ratio": 0.04998822217238082, "last_quartile_mean": 0.00011692704538290855, "mean": 0.00011909236218343722, "normalized": [ 0.9750962012704768, 1.0787587926228024, 0.9990749822765077, 0.9414458855238388, 1.184171880260024, 1.0992323654983946, 1.0501225502428198, 0.9292566224605819, 0.868663497812418, 0.8242103496947312, 0.7928762095957509, 0.775397639822147, 0.9070348138181221, 0.84699741003438, 0.8048391153807885, 0.7619763944335751, 0.8667969819671069, 0.8267630854992756, 0.8140323367256658, 0.8291101141632627, 2.013409927524661, 1.598685759256495, 1.3082952145207551, 1.0492064905857024, 1.337804383430076, 1.2913043056904028, 1.2258631641080262, 1.1663140322183538, 0.9934736018881191, 0.6660773384159757, 0.6009234048133317, 0.5727851484454315 ], "peak_layer": 21, "peak_normalized": 2.013409927524661, "population_cv": 0.2880553317014668, "rest_mean": 0.00010891480929718387, "spike_contrast": 1.598048501322635, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00017405114776920528, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 0.00011612650996539742, 0.00012847193283960223, 0.00011898219963768497, 0.00011211901437491179, 0.00014102582645136863, 0.00013091017899569124, 0.00012506157509051263, 0.00011066736624343321, 0.00010345118789700791, 9.815715748118237e-05, 9.442550071980804e-05, 9.234393655788153e-05, 0.00010802091856021434, 0.00010087092232424766, 9.585019142832607e-05, 9.074556874111295e-05, 0.000103228900115937, 9.846116881817579e-05, 9.694503387436271e-05, 9.874068200588226e-05, 0.000239781744312495, 0.0001903912634588778, 0.00015580796753056347, 0.00012495247938204557, 0.00015932228416204453, 0.00015378448006231338, 0.0001459909399272874, 0.00013889909314457327, 0.00011831511801574379, 7.932472362881526e-05, 7.156538777053356e-05, 6.821433635195717e-05 ] } } }, "post_attention_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "90a633f12d984935028fe5d7212be48e7da7b4a0806751939a551938d16dc540", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.0933690071105957, 0.35791492462158203, 0.3940892517566681, 0.4057151973247528, 0.09380833059549332, 0.11286329478025436, 0.13231655955314636, 0.1621972620487213, 0.05156813934445381, 0.08533531427383423, 0.12003090232610703, 0.16063299775123596, 0.046871740370988846, 0.079384446144104, 0.11459959298372269, 0.15625107288360596, 0.08397205173969269, 0.11654024571180344, 0.15294702351093292, 0.21572710573673248, 0.03708760067820549, 0.06553087383508682, 0.09390194714069366, 0.12852425873279572, 0.025487789884209633, 0.06059965491294861, 0.08860157430171967, 0.12100546807050705, 0.024166326969861984, 0.06877554953098297, 0.12682415544986725, 0.2025238573551178 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.4685406881653762e-05, "first_to_last_ratio": 0.9410126856575548, "imbalance_abs_log_ratio": 0.06079865844877904, "last_quartile_mean": 1.5605960584252898e-05, "mean": 1.5210236995244486e-05, "normalized": [ 0.8983642987423947, 1.0440184727544453, 0.9321347877801589, 0.8651615025486887, 1.0983319803427682, 1.026363548196726, 0.9818143694851882, 0.8777705741109677, 0.8391116720670624, 0.768111080280608, 0.7294629413199059, 0.7043362844025745, 0.8394337872293693, 0.7866999414858254, 0.7506070182724527, 0.7129872919118094, 0.7986271717271862, 0.756519658235185, 0.7558443345590953, 0.7482778272723889, 2.231856850963501, 1.900164458042753, 1.529766702435606, 1.2160980490570046, 1.2050070505856203, 1.1804427049707458, 1.1439999815532713, 1.0872098885199082, 1.376420750807346, 0.9822365813212842, 0.6496057073401301, 0.5832127316780287 ], "peak_layer": 21, "peak_normalized": 2.231856850963501, "population_cv": 0.35072064604634423, "rest_mean": 1.3473513482259241e-05, "spike_contrast": 1.8249541218584802, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.4588543965364806e-05, "top_five_layers": [ 21, 22, 23, 29, 24 ], "top_five_spike_overlap": 4 }, "values": [ 1.366433389193844e-05, 1.587976839800831e-05, 1.417799103364814e-05, 1.3159311492927372e-05, 1.6705889720469713e-05, 1.5611232811352238e-05, 1.4933629245206248e-05, 1.3351098459679633e-05, 1.2763087397615891e-05, 1.1683151569741312e-05, 1.1095304216723889e-05, 1.071312181011308e-05, 1.2767986845574342e-05, 1.1965892554144375e-05, 1.1416910638217814e-05, 1.0844705684576184e-05, 1.214730855281232e-05, 1.1506843293318525e-05, 1.1496571460156702e-05, 1.1381483091099653e-05, 3.39470716426149e-05, 2.890195173677057e-05, 2.326811409147922e-05, 1.8497139535611495e-05, 1.8328442820347846e-05, 1.795481330191251e-05, 1.7400510841980577e-05, 1.653672006796114e-05, 2.0935685824952088e-05, 1.4940051187295467e-05, 9.88065676210681e-06, 8.870803867466748e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 5.824704885526444e-05, "first_to_last_ratio": 0.9350367765544834, "imbalance_abs_log_ratio": 0.06716941725362942, "last_quartile_mean": 6.229385871847626e-05, "mean": 6.0645835219474975e-05, "normalized": [ 0.8976346879816276, 1.0254495893053235, 0.9269591107641313, 0.8659756201788877, 1.0947262616116444, 1.0237827826933272, 0.9755865410195057, 0.8734532748039424, 0.8378628805013905, 0.7669189816721008, 0.7313030327875035, 0.7062404664275104, 0.8453521126718319, 0.791456959252784, 0.7547588398045788, 0.7152101847126882, 0.8029993520923662, 0.7592915987916822, 0.7575926389966431, 0.7514104697092646, 2.2290129358224116, 1.909130875617406, 1.5302419657496316, 1.2102524083356467, 1.1992858934013104, 1.1752325530702608, 1.1403569072303135, 1.0851184586099978, 1.3786752201731949, 0.9879044494381524, 0.6581792561250126, 0.5926436906479273 ], "peak_layer": 21, "peak_normalized": 2.2290129358224116, "population_cv": 0.3496286314628522, "rest_mean": 5.373238053397034e-05, "spike_contrast": 1.8234533729481175, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.79784905212e-05, "top_five_layers": [ 21, 22, 23, 29, 24 ], "top_five_spike_overlap": 4 }, "values": [ 5.443780537461862e-05, 6.218924681888893e-05, 5.621620948659256e-05, 5.2517814765451476e-05, 6.639058847213164e-05, 6.208816193975508e-05, 5.916526060900651e-05, 5.2971303375670686e-05, 5.081289418740198e-05, 4.651044218917377e-05, 4.4350483221933246e-05, 4.283054295228794e-05, 5.126708492753096e-05, 4.799856833415106e-05, 4.57729802292306e-05, 4.3374519009375945e-05, 4.8698566388338804e-05, 4.604787318385206e-05, 4.594483834807761e-05, 4.556991552817635e-05, 0.00013518035120796412, 0.00011578083649510518, 9.280280210077763e-05, 7.339676812989637e-05, 7.273169467225671e-05, 7.127295975806192e-05, 6.915789708727971e-05, 6.58079152344726e-05, 8.361091022379696e-05, 5.9912290453212336e-05, 3.991583071183413e-05, 3.59413716068957e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.051654065027832985, "first_to_last_ratio": 0.9350367766417864, "imbalance_abs_log_ratio": 0.06716941716026084, "last_quartile_mean": 0.05524281645193696, "mean": 0.05378133268095553, "normalized": [ 0.8976347343541998, 1.0254496159807391, 0.9269590662224638, 0.8659756079795379, 1.0947263152911588, 1.0237827665132113, 0.9755865456228415, 0.8734532260217599, 0.8378629005663104, 0.7669189361842735, 0.7313030510773313, 0.7062404620755532, 0.8453520862556849, 0.7914569635579936, 0.7547588454024869, 0.715210168065748, 0.8029993281283683, 0.7592915621352032, 0.7575927111527985, 0.7514104617896089, 2.2290128755662093, 1.9091308586404296, 1.5302420657878328, 1.210252407402922, 1.1992858622982867, 1.1752324985600509, 1.140356942870867, 1.085118449221828, 1.3786752719225042, 0.9879044966912178, 0.6581792432468611, 0.5926436734137177 ], "peak_layer": 21, "peak_normalized": 2.2290128755662093, "population_cv": 0.34962863165324953, "rest_mean": 0.04765041200099168, "spike_contrast": 1.8234533701608295, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.08688830435276032, "top_five_layers": [ 21, 22, 23, 29, 24 ], "top_five_spike_overlap": 4 }, "values": [ 0.04827599227428436, 0.055150046944618225, 0.049853093922138214, 0.046573322266340256, 0.05887584015727043, 0.05506040155887604, 0.05246834456920624, 0.046975478529930115, 0.0450613833963871, 0.041245922446250916, 0.03933045268058777, 0.037982553243637085, 0.0454641617834568, 0.04256561025977135, 0.04059193655848503, 0.03846495598554611, 0.04318637400865555, 0.04083571210503578, 0.040744345635175705, 0.04041185602545738, 0.11987928301095963, 0.10267560184001923, 0.08229845762252808, 0.06508898735046387, 0.06449919193983078, 0.06320556998252869, 0.0613299161195755, 0.058359116315841675, 0.07414699345827103, 0.05313082039356232, 0.03539775684475899, 0.03187316656112671 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 2.4523900208350824e-06, "first_to_last_ratio": 0.5838893617877652, "imbalance_abs_log_ratio": 0.5380437630951902, "last_quartile_mean": 4.200093684403328e-06, "mean": 3.366479703004188e-06, "normalized": [ 0.8052800146448036, 0.7248548976027345, 0.6875613375783711, 0.6714216604086396, 0.7471972903905182, 0.7350070431429925, 0.7286587777774866, 0.7278040529264302, 0.6791389714265288, 0.6550789565378197, 0.6521680291292323, 0.6592890927362068, 0.6955198356719712, 0.6979117144452075, 0.7092924889673354, 0.7222206160250382, 0.7203581195815196, 0.7486202333507752, 0.7725217307187078, 0.7997354810633805, 2.236804466438835, 2.100846413572119, 1.810615219153533, 1.5311186394269638, 1.4728401153035982, 1.4587107807013648, 1.42642683360885, 1.361260935960268, 1.6176028810137273, 1.1674420293359653, 0.7872980356368847, 0.6893933057221917 ], "peak_layer": 21, "peak_normalized": 2.236804466438835, "population_cv": 0.45473851731241766, "rest_mean": 2.848761899561699e-06, "spike_contrast": 2.1630996407743726, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.162155841593631e-06, "top_five_layers": [ 21, 22, 23, 29, 24 ], "top_five_spike_overlap": 4 }, "values": [ 2.7109588245366467e-06, 2.4402093004027847e-06, 2.314661287527997e-06, 2.260327391923056e-06, 2.515424512239406e-06, 2.474386292306008e-06, 2.4530149858037475e-06, 2.4501375719410134e-06, 2.2863075628265506e-06, 2.2053100110497326e-06, 2.1955104330118047e-06, 2.2194833491084864e-06, 2.3414534098264994e-06, 2.349505621168646e-06, 2.387818767601857e-06, 2.4313410449394723e-06, 2.4250709884654498e-06, 2.520214820833644e-06, 2.6006787265941966e-06, 2.692293264772161e-06, 7.5301568358554505e-06, 7.0724568104196806e-06, 6.0953993852308486e-06, 5.154479822522262e-06, 4.958286353939911e-06, 4.910720235784538e-06, 4.802036983164726e-06, 4.582657311402727e-06, 5.445627266453812e-06, 3.930169896193547e-06, 2.6504228571866406e-06, 2.3208285711007193e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 4.423062136993394e-05, "first_to_last_ratio": 1.134588964454276, "imbalance_abs_log_ratio": 0.1262704394874562, "last_quartile_mean": 3.898382828992908e-05, "mean": 4.2956231936841505e-05, "normalized": [ 0.9773911352248565, 1.1018781749688682, 1.0009182870023596, 0.9371353504311951, 1.1786917399256818, 1.0987539475368406, 1.029066314278542, 0.9135023357019427, 0.886011590320582, 0.807852041450046, 0.7639934439920886, 0.7314461758930093, 0.9018893392757595, 0.8344991861415064, 0.7871014631710255, 0.7368850755924464, 0.8618149612362159, 0.8084696882557325, 0.8061263906461698, 0.8009709495863369, 2.283271791494278, 1.904106268279819, 1.4813558668554456, 1.1066733432313076, 1.1285543684542887, 1.095784872846897, 1.0608399359262368, 1.0103793860357428, 1.1008136171260765, 0.7826004245337264, 0.5651158305309029, 0.5161067040540737 ], "peak_layer": 21, "peak_normalized": 2.283271791494278, "population_cv": 0.3420813023936581, "rest_mean": 3.833611157865057e-05, "spike_contrast": 1.7713033240671414, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.790488187107258e-05, "top_five_layers": [ 21, 22, 23, 5, 25 ], "top_five_spike_overlap": 4 }, "values": [ 4.198504029773176e-05, 4.733253445010632e-05, 4.299567808629945e-05, 4.025580346933566e-05, 5.063215576228686e-05, 4.719832941191271e-05, 4.420481127453968e-05, 3.9240618207259104e-05, 3.805971937254071e-05, 3.470227966317907e-05, 3.2818279578350484e-05, 3.1420171580975875e-05, 3.874176763929427e-05, 3.584694059100002e-05, 3.381091300980188e-05, 3.165380621794611e-05, 3.702032336150296e-05, 3.472881144261919e-05, 3.462815220700577e-05, 3.440669388510287e-05, 9.808075265027583e-05, 8.179323049262166e-05, 6.363346619764343e-05, 4.7538516810163856e-05, 4.8478443204658106e-05, 4.707078915089369e-05, 4.5569686335511506e-05, 4.340209125075489e-05, 4.728680505650118e-05, 3.3617565350141376e-05, 2.427524668746628e-05, 2.2169999283505604e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 3.197587261638546e-05, "first_to_last_ratio": 1.333131635512994, "imbalance_abs_log_ratio": 0.28753078764355694, "last_quartile_mean": 2.39855328345584e-05, "mean": 2.970248812061982e-05, "normalized": [ 1.0361629181703018, 1.1578304999935727, 1.0541566667868572, 0.98635734245228, 1.2437452574288521, 1.1620042716757908, 1.0492071029329346, 0.9228441045175485, 0.9216700054083035, 0.8307285213359985, 0.7793058953895384, 0.7399010667098345, 0.943289302505222, 0.863801225057549, 0.804055980550401, 0.7468425326932846, 0.9174307066007519, 0.8500344047071273, 0.8539840368765227, 0.8539730748620978, 2.4143872290296677, 1.962475673509238, 1.450448675843678, 0.9951551245320913, 1.0697087644809422, 1.0364534421727147, 1.0075338109106136, 0.9700903866890204, 0.8555715080381047, 0.6052495605831162, 0.4719782271887709, 0.4436226803672735 ], "peak_layer": 21, "peak_normalized": 2.4143872290296677, "population_cv": 0.36965707435492423, "rest_mean": 2.6520828584950693e-05, "spike_contrast": 1.7677973168544674, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.68834496132331e-05, "top_five_layers": [ 21, 22, 23, 5, 6 ], "top_five_spike_overlap": 3 }, "values": [ 3.077661676798016e-05, 3.4390446671750396e-05, 3.131107587250881e-05, 2.929726724687498e-05, 3.694232873385772e-05, 3.451441807555966e-05, 3.1164061510935426e-05, 2.741076605161652e-05, 2.737589238677174e-05, 2.4674704036442563e-05, 2.3147324100136757e-05, 2.197690264438279e-05, 2.801803930196911e-05, 2.5657045625848696e-05, 2.3882463210611604e-05, 2.2183081455295905e-05, 2.724997466430068e-05, 2.5248136807931587e-05, 2.5365450710523874e-05, 2.536512511142064e-05, 7.171330798882991e-05, 5.8290410379413515e-05, 4.3081934563815594e-05, 2.955858326458838e-05, 3.1773011869518086e-05, 3.078524605371058e-05, 2.9926261049695313e-05, 2.8814098186558113e-05, 2.541260255384259e-05, 1.7977417883230373e-05, 1.401892768626567e-05, 1.3176697393646464e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00012400272680679336, "first_to_last_ratio": 0.9335381164617736, "imbalance_abs_log_ratio": 0.06877348505915627, "last_quartile_mean": 0.00013283091993798735, "mean": 0.00012800787476408004, "normalized": [ 0.9126506619620831, 1.0021232925866939, 0.9288016689522621, 0.8788641028994991, 1.1046193134946338, 1.0349212917262334, 1.0027880923141508, 0.8849252279796427, 0.8585273472149247, 0.7923749429396013, 0.7562228200041292, 0.736680174869512, 0.8610462651973099, 0.8137835290329183, 0.777808974013706, 0.7321345726290727, 0.8100576721472483, 0.7698492818596471, 0.7757169806852989, 0.7602388446914977, 2.0631811719563116, 1.8046577463407538, 1.4496405955781522, 1.1869636495039606, 1.265965684421126, 1.2453010760665815, 1.2011262255035766, 1.1413138029656218, 1.3511026794770848, 0.9237696874994554, 0.6138670080500801, 0.5589756154372304 ], "peak_layer": 21, "peak_normalized": 2.0631811719563116, "population_cv": 0.3209165643286295, "rest_mean": 0.0001148732766590978, "spike_contrast": 1.7317753120366732, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00019893470453098417, "top_five_layers": [ 21, 22, 23, 29, 25 ], "top_five_spike_overlap": 4 }, "values": [ 0.00011682647163979709, 0.00012827967293560505, 0.0001188939277199097, 0.00011250152601860464, 0.00014139997074380517, 0.00013247807510197163, 0.00012836477253586054, 0.00011327739775879309, 0.00010989826114382595, 0.00010143023246200755, 9.680247603682801e-05, 9.430086356587708e-05, 0.0001102207024814561, 0.0001041707000695169, 9.956567373592407e-05, 9.37189906835556e-05, 0.00010369376104790717, 9.854677045950666e-05, 9.929788211593404e-05, 9.731655882205814e-05, 0.0002641034370753914, 0.00023101040278561413, 0.0001855654118116945, 0.00015194069419521838, 0.00016205357678700238, 0.0001594083441887051, 0.000153753615450114, 0.00014609715435653925, 0.00017295178258791566, 0.00011824979446828365, 7.857981108827516e-05, 7.155328057706356e-05 ] } } }, "post_mlp_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "89f2226456bfe96e245d2aad0f699d32c676f9f53099789cd870253d002402f5", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.16915453970432281, 0.374446302652359, 0.39966297149658203, 0.4194087088108063, 0.09892481565475464, 0.11664740741252899, 0.13951337337493896, 0.18593810498714447, 0.06607726216316223, 0.10425819456577301, 0.14551182091236115, 0.19865906238555908, 0.06484340131282806, 0.10430748760700226, 0.14720691740512848, 0.20439592003822327, 0.10052042454481125, 0.14085541665554047, 0.1745898574590683, 0.2420409768819809, 0.049456026405096054, 0.08509506285190582, 0.11783231794834137, 0.15649692714214325, 0.05533855780959129, 0.08507220447063446, 0.11942370235919952, 0.1559232622385025, 0.06628894805908203, 0.12384890764951706, 0.19928614795207977, 0.30097496509552 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.4557187910213543e-05, "first_to_last_ratio": 1.0607633444033049, "imbalance_abs_log_ratio": 0.05898878517983358, "last_quartile_mean": 1.3723313486480038e-05, "mean": 1.4120565339226232e-05, "normalized": [ 0.9661042073868427, 1.1234927272170097, 1.0048805588748393, 0.930122829905949, 1.1780606109949872, 1.0933476630445367, 1.0326811915847935, 0.9186785339996938, 0.847834760542847, 0.8013801206656552, 0.7660112105266416, 0.7425304402788336, 0.8825126884919682, 0.8223586568441924, 0.7771880149121451, 0.7449670413455409, 0.8551700697969584, 0.802518746984906, 0.798912087896052, 0.8260410611842874, 2.177573691326528, 1.6956266245403484, 1.356783978848815, 1.0802853429406405, 1.2722390101630212, 1.2327680051591792, 1.1723280609516593, 1.1186700219177557, 1.0579152454074585, 0.703985253507422, 0.6306380222059869, 0.5863935205525064 ], "peak_layer": 21, "peak_normalized": 2.177573691326528, "population_cv": 0.3121818474280679, "rest_mean": 1.276995489105401e-05, "spike_contrast": 1.6768940800533054, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.1413861759356223e-05, "top_five_layers": [ 21, 22, 23, 25, 26 ], "top_five_spike_overlap": 4 }, "values": [ 1.3641937584907282e-05, 1.5864352462813258e-05, 1.418948158971034e-05, 1.3133860193192959e-05, 1.6634881831123494e-05, 1.5438687114510685e-05, 1.458204224036308e-05, 1.2972260265087243e-05, 1.1971906133112498e-05, 1.1315940355416387e-05, 1.0816511348821223e-05, 1.0484949598321691e-05, 1.2461578080547042e-05, 1.1612169146246742e-05, 1.0974334145430475e-05, 1.0519355782889761e-05, 1.207548484671861e-05, 1.1332018402754329e-05, 1.1281090337433852e-05, 1.1664166777336504e-05, 3.074857158935629e-05, 2.3943206542753614e-05, 1.9158556824550033e-05, 1.5254239770001732e-05, 1.7964734070119448e-05, 1.740738116495777e-05, 1.65539349836763e-05, 1.579625313752331e-05, 1.4938361346139573e-05, 9.940669770003296e-06, 8.90496539796004e-06, 8.280208021460567e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 5.775084036940825e-05, "first_to_last_ratio": 1.052422517064877, "imbalance_abs_log_ratio": 0.05109466587384658, "last_quartile_mean": 5.487419684868655e-05, "mean": 5.6330586062358634e-05, "normalized": [ 0.9642721910669776, 1.1028139338894607, 0.9987938805714334, 0.9304696940088941, 1.1736512898896914, 1.0906863445980115, 1.0265157914934855, 0.9144996304913484, 0.8458285704886631, 0.7999991684983165, 0.7680400541626473, 0.7450107835374574, 0.8894180522547344, 0.828322343938663, 0.7813593871872957, 0.7472680764959349, 0.8590598127718557, 0.8034697757967721, 0.7989535107307866, 0.8287435520147443, 2.1774545886808907, 1.7073577010754475, 1.349137992598525, 1.075708464409634, 1.26611104795802, 1.2278203792888875, 1.1682950628124344, 1.1168114493745398, 1.0638792469844063, 0.7127724138014546, 0.6406649107248228, 0.596810898403764 ], "peak_layer": 21, "peak_normalized": 2.1774545886808907, "population_cv": 0.3104742799843349, "rest_mean": 5.09567111698238e-05, "spike_contrast": 1.6749415047137133, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 8.534951048204676e-05, "top_five_layers": [ 21, 22, 23, 25, 26 ], "top_five_spike_overlap": 4 }, "values": [ 5.431801764643751e-05, 6.212215521372855e-05, 5.626264464808628e-05, 5.2413903176784515e-05, 6.611246499232948e-05, 6.143900100141764e-05, 5.7824236137093976e-05, 5.151430013938807e-05, 4.7646019083913416e-05, 4.5064422010909766e-05, 4.326414637034759e-05, 4.1966894059441984e-05, 5.0101440137950704e-05, 4.6659883082611486e-05, 4.401443220558576e-05, 4.209404869470745e-05, 4.8391342716058716e-05, 4.525992335402407e-05, 4.5005519496044144e-05, 4.668360998039134e-05, 0.00012265729310456663, 9.617645991966128e-05, 7.599773380206898e-05, 6.059528823243454e-05, 7.132077735150233e-05, 6.91638415446505e-05, 6.581074558198452e-05, 6.291064346441999e-05, 5.9928941482212394e-05, 4.015088779851794e-05, 3.608902989071794e-05, 3.36187076754868e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.05121402395889163, "first_to_last_ratio": 1.0524225347723017, "imbalance_abs_log_ratio": 0.051094682699241514, "last_quartile_mean": 0.048662986839190125, "mean": 0.049954527348745614, "normalized": [ 0.9642721981903206, 1.1028138895874178, 0.9987938962581453, 0.9304696881830206, 1.173651303472138, 1.0906864280408863, 1.026515821852437, 0.9144996713487815, 0.8458286213336422, 0.7999991004556222, 0.7680400505122134, 0.7450108195554819, 0.8894180250248003, 0.8283223877794939, 0.7813593477683367, 0.7472680886765953, 0.8590597725624388, 0.8034697570645826, 0.7989535036289304, 0.8287435796258069, 2.1774546085624773, 1.7073576758392675, 1.3491378883786493, 1.0757084641690238, 1.2661110422632107, 1.2278203930777054, 1.1682951272735569, 1.116811433982744, 1.0638792225575855, 0.7127724168046565, 0.6406648649018053, 0.5968109112682259 ], "peak_layer": 21, "peak_normalized": 2.1774546085624773, "population_cv": 0.3104742795822218, "rest_mean": 0.0451889215382161, "spike_contrast": 1.6749414712540835, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.07568879872560501, "top_five_layers": [ 21, 22, 23, 25, 26 ], "top_five_spike_overlap": 4 }, "values": [ 0.04816976189613342, 0.05509054660797119, 0.04989427700638771, 0.0464811734855175, 0.058629196137189865, 0.05448472499847412, 0.051279112696647644, 0.045683398842811584, 0.04225296899676323, 0.039963576942682266, 0.038367077708244324, 0.0372166633605957, 0.044430457055568695, 0.041378453373909, 0.03903243690729141, 0.03732942417263985, 0.042913924902677536, 0.040136951953172684, 0.03991134464740753, 0.04139949381351471, 0.10877371579408646, 0.08529024571180344, 0.06739554554224014, 0.05373650789260864, 0.06324797868728638, 0.061335187405347824, 0.05836163088679314, 0.05578978732228279, 0.05314558371901512, 0.03560620918869972, 0.032004110515117645, 0.029813406988978386 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 2.436454508369934e-06, "first_to_last_ratio": 0.657100210816764, "imbalance_abs_log_ratio": 0.4199187442493236, "last_quartile_mean": 3.7078887942243455e-06, "mean": 3.0860571129665004e-06, "normalized": [ 0.8748782196381274, 0.7901803669223133, 0.7516544304960662, 0.734260150489015, 0.8140562976628272, 0.7953559325779443, 0.7852756405934781, 0.7703711520364845, 0.7053050982383477, 0.6974195920824781, 0.6980807023325704, 0.711441053073735, 0.7419051725418756, 0.7402238467971856, 0.7479616291091419, 0.7670711266155963, 0.7814113911343411, 0.799879155633072, 0.8213887051173213, 0.8665755501883736, 2.1095153525815307, 1.8446758978581441, 1.6136789811259926, 1.4254574374393147, 1.5802379867503822, 1.5432560407845088, 1.4958212882432333, 1.4165316754469297, 1.2619021654775193, 0.8534679877307335, 0.7626784604938425, 0.6980815127875744 ], "peak_layer": 21, "peak_normalized": 2.1095153525815307, "population_cv": 0.3913235859713044, "rest_mean": 2.6776042347962545e-06, "spike_contrast": 1.9762825985702426, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.291702655085828e-06, "top_five_layers": [ 21, 22, 23, 25, 26 ], "top_five_spike_overlap": 4 }, "values": [ 2.6999241526937112e-06, 2.438541741867084e-06, 2.319648501725169e-06, 2.265968760184478e-06, 2.5122242277575424e-06, 2.4545138330722693e-06, 2.423405476292828e-06, 2.3774093733663904e-06, 2.1766118152299896e-06, 2.152276692868327e-06, 2.154316916858079e-06, 2.1955477222945774e-06, 2.289561734869494e-06, 2.2843730675958795e-06, 2.3082523057382787e-06, 2.367225306443288e-06, 2.4114801817631815e-06, 2.4684727577550802e-06, 2.534852455937653e-06, 2.674301640581689e-06, 6.510084858746268e-06, 5.692775175702991e-06, 4.979905497748405e-06, 4.3990430640405975e-06, 4.87670467919088e-06, 4.762576281791553e-06, 4.616189926309744e-06, 4.371497652755352e-06, 3.894302153639728e-06, 2.633850954225636e-06, 2.3536692879133625e-06, 2.1543194179685088e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 4.390457525005331e-05, "first_to_last_ratio": 1.2504647302280618, "imbalance_abs_log_ratio": 0.22351526640204505, "last_quartile_mean": 3.511060663186072e-05, "mean": 4.0363916752994555e-05, "normalized": [ 1.0382164357293802, 1.1714403344120268, 1.066170093792947, 0.9959174993218778, 1.2500532492710323, 1.1579732777162342, 1.0717368509641163, 0.9502395185988469, 0.8869414048946417, 0.833690474916492, 0.7927071585210932, 0.7629637092538301, 0.9387915425025706, 0.8642303082645213, 0.8052906759294536, 0.7607410261601476, 0.9118868113212986, 0.8474155723120622, 0.8424653007805989, 0.8780425608667576, 2.2287682756196885, 1.7083188409707202, 1.2963467242094948, 0.9808417215466828, 1.1774676532233859, 1.134117266308163, 1.0794337282535327, 1.032113768673168, 0.8401118398121776, 0.6107121255088265, 0.5549546947060829, 0.529899555638148 ], "peak_layer": 21, "peak_normalized": 2.2287682756196885, "population_cv": 0.3167604797981032, "rest_mean": 3.678835660307582e-05, "spike_contrast": 1.6220333570857757, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.967194156255573e-05, "top_five_layers": [ 21, 22, 23, 5, 25 ], "top_five_spike_overlap": 4 }, "values": [ 4.190648178337142e-05, 4.7283920139307156e-05, 4.3034800910390913e-05, 4.0199131035478786e-05, 5.0457045290386304e-05, 4.674033698393032e-05, 4.3259497033432126e-05, 3.835538882412948e-05, 3.580042903195135e-05, 3.365101292729378e-05, 3.199676575604826e-05, 3.079620364587754e-05, 3.789330366998911e-05, 3.488372021820396e-05, 3.250468580517918e-05, 3.070648745051585e-05, 3.680732334032655e-05, 3.420501161599532e-05, 3.400519926799461e-05, 3.544123683241196e-05, 8.996181713882834e-05, 6.89544394845143e-05, 5.232563125900924e-05, 3.959061359637417e-05, 4.752720633405261e-05, 4.577741492539644e-05, 4.3570173147600144e-05, 4.166015423834324e-05, 3.3910204365383834e-05, 2.4650733394082636e-05, 2.240014509879984e-05, 2.138882155122701e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 3.179541454301216e-05, "first_to_last_ratio": 1.4442117356867925, "imbalance_abs_log_ratio": 0.36756366108335176, "last_quartile_mean": 2.2015756940163556e-05, "mean": 2.8184001337194786e-05, "normalized": [ 1.0900710410597174, 1.2202377204007206, 1.1129962183013473, 1.0397229089182578, 1.311573392339908, 1.2115346593926428, 1.0850219645529546, 0.9539380457790582, 0.9148379074713061, 0.8482601490653242, 0.80423710953165, 0.7603474091950175, 0.9728586584040144, 0.8839356127299847, 0.8178719137162765, 0.7667897060534457, 0.9607918483104477, 0.8823679413378527, 0.8855291645761406, 0.9341904203414721, 2.366777717788271, 1.761471460320249, 1.281074600338134, 0.8844123035863982, 1.1022706094922554, 1.0641774209193755, 1.0295319476930451, 0.98889434261282, 0.6398981626723077, 0.5095123707592054, 0.46869799120623235, 0.44616728113416654 ], "peak_layer": 21, "peak_normalized": 2.366777717788271, "population_cv": 0.3534591649379896, "rest_mean": 2.5682925196532677e-05, "spike_contrast": 1.623250162423029, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.1689812496770175e-05, "top_five_layers": [ 21, 22, 5, 23, 2 ], "top_five_spike_overlap": 3 }, "values": [ 3.072256367886439e-05, 3.439118154346943e-05, 3.136868690489791e-05, 2.930355185526423e-05, 3.696538624353707e-05, 3.414589446038008e-05, 3.058026049984619e-05, 2.6885791157837957e-05, 2.578379280748777e-05, 2.3907365175546147e-05, 2.2666619770461693e-05, 2.1429632397484966e-05, 2.7419049729360268e-05, 2.491284249117598e-05, 2.3050903109833598e-05, 2.161120210075751e-05, 2.707895873754751e-05, 2.486865923856385e-05, 2.495775515853893e-05, 2.632922405609861e-05, 6.670526636298746e-05, 4.964531399309635e-05, 3.6105808248976246e-05, 2.492627754691057e-05, 3.106639633188024e-05, 2.9992777854204178e-05, 2.9016329790465534e-05, 2.7870999474544078e-05, 1.8034890672424808e-05, 1.436009733879473e-05, 1.3209784810896963e-05, 1.2574779248097911e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00012292819883441553, "first_to_last_ratio": 1.0513192186692275, "imbalance_abs_log_ratio": 0.05004577429361062, "last_quartile_mean": 0.00011692756743286736, "mean": 0.00011909139175259043, "normalized": [ 0.9752660502479068, 1.0789286309694868, 0.9994668646978057, 0.9411063508320493, 1.184341722240097, 1.099393450721871, 1.0501926916398412, 0.9290429064328071, 0.8686779687888649, 0.8241854794593988, 0.792855421816483, 0.775444525710051, 0.9071997093072319, 0.8468645252105945, 0.8047375956308107, 0.7620074082972532, 0.8667004270657389, 0.8267511272325375, 0.8138955786427265, 0.8288750539645799, 2.0135275083229365, 1.598676058788305, 1.3079546979331442, 1.049263794446366, 1.3377524784376515, 1.2915549336704384, 1.2258380843419896, 1.1663823100192856, 0.9933892597178439, 0.6660588776574, 0.6008786919008721, 0.5727898158556317 ], "peak_layer": 21, "peak_normalized": 2.0135275083229365, "population_cv": 0.2880861503224607, "rest_mean": 0.00010891492409562623, "spike_contrast": 1.5979841013100087, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00017404431710019708, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 0.000116145791253075, 0.00012849111226387322, 0.00011902789992745966, 0.00011207766510779038, 0.00014104490401223302, 0.00013092829613015056, 0.00012506890925578773, 0.00011064101272495463, 0.00010345206828787923, 9.815339581109583e-05, 9.442225564271212e-05, 9.234876779373735e-05, 0.0001080396759789437, 0.0001008542749332264, 9.583732025930658e-05, 9.07485227799043e-05, 0.00010321656009182334, 9.845894237514585e-05, 9.692795720184222e-05, 9.871188376564533e-05, 0.00023979379329830408, 0.00019038855680264533, 0.00015576614532619715, 0.00012495828559622169, 0.00015931480447761714, 0.00015381307457573712, 0.00014598676352761686, 0.00013890609261579812, 0.00011830410949187353, 7.932187872938812e-05, 7.155947969295084e-05, 6.821433635195717e-05 ] } } }, "pre_attention_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "ee695e78937de01b7128d3866b8fc5dd328ebdf2c159be37918d4f02952d5248", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.0626930519938469, 0.09030644595623016, 0.20229491591453552, 0.20300470292568207, 0.19049881398677826, 0.12695962190628052, 0.12132706493139267, 0.11585728824138641, 0.13427142798900604, 0.10485932976007462, 0.10419609397649765, 0.10852954536676407, 0.1286265105009079, 0.09096142649650574, 0.10080574452877045, 0.10566592961549759, 0.1287592649459839, 0.09650235623121262, 0.1065765842795372, 0.11446042358875275, 0.12395420670509338, 0.07221997529268265, 0.07677769660949707, 0.08416415005922318, 0.09094838052988052, 0.059439923614263535, 0.062182098627090454, 0.06896103918552399, 0.0776243805885315, 0.06285123527050018, 0.07231374084949493, 0.08016443997621536 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 9.149274319497636e-06, "first_to_last_ratio": 1.5837096584205628, "imbalance_abs_log_ratio": 0.45976998013972153, "last_quartile_mean": 5.777115944738398e-06, "mean": 9.634552974091548e-06, "normalized": [ 0.8630079758667576, 1.1841398261001996, 0.8051062054139604, 0.7156998136677282, 0.4445065896427089, 1.0024512634543776, 1.1807728875805437, 1.4013668793534428, 0.8789821247819359, 0.7223863966761895, 0.6538379941247476, 0.5671559651137404, 0.5188060313049182, 0.9822467059592832, 0.7788800605198396, 0.6818489458877146, 0.890080362955609, 1.2874303670749199, 1.3369973433842235, 1.4811521925961615, 1.7948063939957373, 3.2206785668752556, 2.086273370175758, 1.7243881308393287, 0.6329011343455747, 0.8449042739736187, 0.6890450017195305, 0.6840178633584191, 0.6408403483767343, 0.3915727928128283, 0.4376587254931133, 0.4760574665750997 ], "peak_layer": 22, "peak_normalized": 3.2206785668752556, "population_cv": 0.5757500479814839, "rest_mean": 8.043407408169716e-06, "spike_contrast": 2.266046975508971, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.822673903006944e-05, "top_five_layers": [ 22, 23, 21, 24, 20 ], "top_five_spike_overlap": 4 }, "values": [ 8.314696060551796e-06, 1.1408657883293927e-05, 7.756838385830633e-06, 6.895447768329177e-06, 4.282622285245452e-06, 9.658169801696204e-06, 1.1376218935765792e-05, 1.3501543435268104e-05, 8.46859984449111e-06, 6.959870006539859e-06, 6.29943679086864e-06, 5.46429419046035e-06, 4.998464191885432e-06, 9.463507922191638e-06, 7.504161203542026e-06, 6.569309789483668e-06, 8.575526408094447e-06, 1.2403816072037444e-05, 1.2881371731054969e-05, 1.4270239262259565e-05, 1.7292157281190157e-05, 3.10297982650809e-05, 2.0100311303394847e-05, 1.661370879446622e-05, 6.097719506215071e-06, 8.140274985635187e-06, 6.638640570599819e-06, 6.590206339751603e-06, 6.174210284370929e-06, 3.7726288155681686e-06, 4.216646175336791e-06, 4.586600880429614e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 3.508693453113665e-05, "first_to_last_ratio": 1.545538855420658, "imbalance_abs_log_ratio": 0.4353726232773416, "last_quartile_mean": 2.2702072101310478e-05, "mean": 3.798078637373692e-05, "normalized": [ 0.7322198380142775, 1.1146512330351426, 0.7682377165905682, 0.7121731986404677, 0.45017436859089877, 0.9653491070005965, 1.2078065620492038, 1.4398478124552039, 0.8563055841321604, 0.7225599015907835, 0.6267744679735274, 0.5719403525829381, 0.5281463834699767, 0.9983424395163674, 0.7804786200583431, 0.723763963353606, 0.8874153102228961, 1.359191814070613, 1.3289879240318254, 1.530666377556468, 1.8861456778359527, 3.1337276091629214, 2.1560403772085897, 1.7372519778271556, 0.617860933943478, 0.884112844811131, 0.6705334756654291, 0.6235488697957358, 0.6422635269195309, 0.40758134823751485, 0.4461421673261394, 0.4897582163305576 ], "peak_layer": 22, "peak_normalized": 3.1337276091629214, "population_cv": 0.5791745563768504, "rest_mean": 3.160701035759067e-05, "spike_contrast": 2.2906050284993005, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 7.239917686092667e-05, "top_five_layers": [ 22, 23, 21, 24, 20 ], "top_five_spike_overlap": 4 }, "values": [ 2.7810285246232525e-05, 4.23353303631302e-05, 2.9178272598073818e-05, 2.704889811866451e-05, 1.709797652438283e-05, 3.666471820906736e-05, 4.5873443013988435e-05, 5.4686552175553516e-05, 3.252315946156159e-05, 2.7443393264547922e-05, 2.3805387172615156e-05, 2.1722744349972345e-05, 2.0059414964634925e-05, 3.791783092310652e-05, 2.964319173770491e-05, 2.7489124477142468e-05, 3.3704731322359294e-05, 5.16231739311479e-05, 5.0476006435928866e-05, 5.8135912695433944e-05, 7.163729605963454e-05, 0.00011902143887709826, 8.188810897991061e-05, 6.598219624720514e-05, 2.3466844140784815e-05, 3.3579301089048386e-05, 2.5467388695687987e-05, 2.368287641729694e-05, 2.4393673811573535e-05, 1.5480260117328726e-05, 1.6944830349530093e-05, 1.860140218923334e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.031115445191971958, "first_to_last_ratio": 1.5455389044638999, "imbalance_abs_log_ratio": 0.4353726550094724, "last_quartile_mean": 0.02013242442626506, "mean": 0.03368174110073596, "normalized": [ 0.7322198864053264, 1.1146512158984638, 0.7682377545837001, 0.712173208832108, 0.4501743829001258, 0.9653491219926751, 1.2078066415745399, 1.439847823453456, 0.8563056236135416, 0.7225599050210681, 0.6267744585945153, 0.571940312180098, 0.5281463577956852, 0.9983424497176008, 0.780478593834227, 0.7237639256228467, 0.8874152773655765, 1.3591918260566556, 1.328987897521226, 1.530666325439014, 1.8861457212289645, 3.1337276246878876, 2.1560403331002465, 1.7372519723591637, 0.6178609345905073, 0.884112792417156, 0.6705334625370614, 0.6235488977990011, 0.6422635321867042, 0.4075813522428957, 0.4461421692244057, 0.48975821922355667 ], "peak_layer": 22, "peak_normalized": 3.1337276246878876, "population_cv": 0.5791745569038685, "rest_mean": 0.028029412790029136, "spike_contrast": 2.2906050319181905, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.06420431397855282, "top_five_layers": [ 22, 23, 21, 24, 20 ], "top_five_spike_overlap": 4 }, "values": [ 0.0246624406427145, 0.037543393671512604, 0.02587558515369892, 0.023987233638763428, 0.015162657015025616, 0.03251463919878006, 0.040681030601263046, 0.04849658161401749, 0.028841864317655563, 0.024337075650691986, 0.021110855042934418, 0.019263945519924164, 0.01778888888657093, 0.03362591192126274, 0.026287877932190895, 0.024377629160881042, 0.02988969162106514, 0.04577994719147682, 0.04476262629032135, 0.05155550688505173, 0.06352867186069489, 0.1055494025349617, 0.07261919230222702, 0.05851367115974426, 0.020810632035136223, 0.029778458178043365, 0.022584734484553337, 0.021002212539315224, 0.02163255400955677, 0.013728049583733082, 0.015026845037937164, 0.01649590954184532 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 2.7692196482576037e-06, "first_to_last_ratio": 0.6571523685326055, "imbalance_abs_log_ratio": 0.4198393718110443, "last_quartile_mean": 4.213968907151866e-06, "mean": 3.214015457331243e-06, "normalized": [ 3.0400829135272565, 1.2865160907100315, 0.2472632790010475, 0.3401182550101428, 0.26497135139292444, 0.4638808866565918, 0.5467998258617136, 0.703226962509653, 0.49981781652492185, 0.5324797039826876, 0.44510627131354047, 0.5552007970738991, 0.40945666992731405, 0.724088149295621, 0.6462390031908897, 0.711844411662313, 0.4687224582419591, 0.9339518397555295, 0.7415086809721194, 0.8888770293720882, 1.1099018555296571, 2.1635839654515356, 1.9886856362993341, 1.7986938425863102, 0.9387406713720655, 1.4577052896599199, 1.622147026842953, 1.8026317602590465, 1.7204241916194283, 1.2632366472445733, 0.8228931382269594, 0.8612035789259724 ], "peak_layer": 1, "peak_normalized": 3.0400829135272565, "population_cv": 0.6423947042687261, "rest_mean": 2.85694953298069e-06, "spike_contrast": 1.7998817933123723, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.1421714488242285e-06, "top_five_layers": [ 1, 22, 23, 28, 24 ], "top_five_spike_overlap": 3 }, "values": [ 9.770873475645203e-06, 4.134882601647405e-06, 7.947080007397744e-07, 1.0931453289231285e-06, 8.516220191268076e-07, 1.4909203400748083e-06, 1.7574230923855794e-06, 2.260182327518123e-06, 1.60642218816065e-06, 1.7113979993155226e-06, 1.430578436156793e-06, 1.7844239437181386e-06, 1.316000066253764e-06, 2.327230504306499e-06, 2.077022145385854e-06, 2.2878789422975387e-06, 1.5064812259879545e-06, 3.001735649377224e-06, 2.383220362389693e-06, 2.8568645120685687e-06, 3.5672417197929462e-06, 6.953792308195261e-06, 6.391666374838678e-06, 5.781029813078931e-06, 3.0171270282153273e-06, 4.685087333200499e-06, 5.213605618337169e-06, 5.793686341348803e-06, 5.529469945031451e-06, 4.060062110511353e-06, 2.6447912659932626e-06, 2.7679216145770624e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 2.3451681158803694e-05, "first_to_last_ratio": 1.6385764424162144, "imbalance_abs_log_ratio": 0.49382784196912244, "last_quartile_mean": 1.4312228927337856e-05, "mean": 2.555020634531502e-05, "normalized": [ 0.6071808785596028, 1.1274239496900784, 0.7995305843862176, 0.7321942050911809, 0.5118543228359549, 1.0424113365543313, 1.2752146128472674, 1.2471229511562785, 0.9112265954633286, 0.7814840770627814, 0.6624794896612818, 0.592024262862305, 0.5809674239482662, 1.0556218628866896, 0.8291728241302284, 0.7247433856561432, 0.8865919112332509, 1.3479549488775053, 1.3297886953518112, 1.4968777903072936, 1.8162303752031244, 3.162513433584892, 2.2727498854250054, 1.725352262746421, 0.6264022086117336, 0.8749505442611616, 0.626244872638493, 0.5617433885717087, 0.6521775430514312, 0.4059210443479573, 0.34039452709442314, 0.39345380590185147 ], "peak_layer": 22, "peak_normalized": 3.162513433584892, "population_cv": 0.5868651605559962, "rest_mean": 2.1194134475350706e-05, "spike_contrast": 2.31540450496789, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.90729944431223e-05, "top_five_layers": [ 22, 23, 21, 24, 20 ], "top_five_spike_overlap": 4 }, "values": [ 1.551359673612751e-05, 2.880591455323156e-05, 2.0428171410458162e-05, 1.8707713024923578e-05, 1.3077983567200135e-05, 2.6633824745658785e-05, 3.2581996492808685e-05, 3.186424874002114e-05, 2.3282027541426942e-05, 1.996707942453213e-05, 1.6926487660384737e-05, 1.5126342077564914e-05, 1.4843837561784312e-05, 2.6971356419380754e-05, 2.1185536752454937e-05, 1.851734305091668e-05, 2.2652606276096776e-05, 3.444052708800882e-05, 3.3976375561906025e-05, 3.8245536416070536e-05, 4.640506085706875e-05, 8.08028707979247e-05, 5.806922854389995e-05, 4.408310633152723e-05, 1.6004705685190856e-05, 2.235516694781836e-05, 1.600068571860902e-05, 1.4352659491123632e-05, 1.6663270798744634e-05, 1.0371366442996077e-05, 8.697150406078435e-06, 1.0052825928141829e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 1.5373112091765506e-05, "first_to_last_ratio": 1.6912717159525321, "imbalance_abs_log_ratio": 0.5254807405993072, "last_quartile_mean": 9.089676098028576e-06, "mean": 1.6939541396254754e-05, "normalized": [ 0.5146403958063405, 1.129533445050957, 0.8187597118962306, 0.7436163957619786, 0.58408638410117, 1.1036998747972189, 1.332871469729194, 1.0330181965818857, 0.9704852531314737, 0.8589277684435074, 0.6990872638515552, 0.6230043616382759, 0.6364751804079726, 1.1167915840066862, 0.891220751382709, 0.724153917102275, 0.891409688720543, 1.3388166335309473, 1.337048386220851, 1.3976541580872286, 1.722563990646236, 3.1806416945911438, 2.3938376810770605, 1.6648944521221352, 0.6175722118356713, 0.8664877854166266, 0.6002517746736175, 0.5139830076948199, 0.6595444884172128, 0.4162815452236349, 0.29121692005268845, 0.3274236280001534 ], "peak_layer": 22, "peak_normalized": 3.1806416945911438, "population_cv": 0.5907005231242171, "rest_mean": 1.4066400665426584e-05, "spike_contrast": 2.307235668503023, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.245450134272687e-05, "top_five_layers": [ 22, 23, 21, 24, 20 ], "top_five_spike_overlap": 4 }, "values": [ 8.717772288946435e-06, 1.913377855089493e-05, 1.3869414033251815e-05, 1.2596520718943793e-05, 9.894155482470524e-06, 1.8696169718168676e-05, 2.2578231437364593e-05, 1.7498854504083283e-05, 1.643957511987537e-05, 1.4549842489941511e-05, 1.1842217645607889e-05, 1.0553408174018841e-05, 1.0781597666209564e-05, 1.891793726827018e-05, 1.5096870811248664e-05, 1.226683525601402e-05, 1.5100071323104203e-05, 2.267893978569191e-05, 2.264898648718372e-05, 2.3675620468566194e-05, 2.91794440272497e-05, 5.387861165218055e-05, 4.055051249451935e-05, 2.8202548492117785e-05, 1.0461390047566965e-05, 1.4677905710414052e-05, 1.0167989785259124e-05, 8.706636435817927e-06, 1.117238116421504e-05, 7.051618467812659e-06, 4.933081072522327e-06, 5.546406100620516e-06 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 7.035670569166541e-05, "first_to_last_ratio": 1.628534826047847, "imbalance_abs_log_ratio": 0.48768073085878644, "last_quartile_mean": 4.320245693634206e-05, "mean": 7.59426430363419e-05, "normalized": [ 0.7176661778769349, 1.1370331704423866, 0.7780076259970639, 0.7262199688673726, 0.4483696422381095, 1.003999492948003, 1.2202063327075847, 1.3800600922709334, 0.8629609118801197, 0.6931775499618317, 0.6127925134987474, 0.5458438175283321, 0.5302145621041596, 0.9769292546483475, 0.771934894275089, 0.72956100604724, 0.8531490582833261, 1.317547094535059, 1.3252811472222463, 1.5818422780190082, 1.9486695160166534, 3.273079379913776, 2.1658432560750196, 1.8485495385860136, 0.64594937576605, 0.8849670913021926, 0.6396728591713527, 0.582207345426845, 0.6424680748091891, 0.36789289300144107, 0.35209350547207957, 0.4358105731074919 ], "peak_layer": 22, "peak_normalized": 3.273079379913776, "population_cv": 0.6112680425269249, "rest_mean": 6.221083195770107e-05, "spike_contrast": 2.412673454729811, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00015009442286100237, "top_five_layers": [ 22, 23, 21, 24, 20 ], "top_five_spike_overlap": 4 }, "values": [ 5.450146636576392e-05, 8.634930418338627e-05, 5.9083955420646816e-05, 5.5151063861558214e-05, 3.405037568882108e-05, 7.624637510161847e-05, 9.266569395549595e-05, 0.00010480541095603257, 6.553553248522803e-05, 5.264173523755744e-05, 4.6537083107978106e-05, 4.1452822188148275e-05, 4.026589522254653e-05, 7.419058965751901e-05, 5.8622776123229414e-05, 5.540479105548002e-05, 6.479039439000189e-05, 0.0001000580086838454, 0.00010064535308629274, 0.00012012928345939144, 0.00014798711345065385, 0.0002485662989784032, 0.00016447986126877367, 0.00014038373774383217, 4.905510286334902e-05, 6.720673991367221e-05, 4.857844760408625e-05, 4.42143646068871e-05, 4.879072366748005e-05, 2.7938758648815565e-05, 2.6738911401480436e-05, 3.309660678496584e-05 ] } } }, "pre_mlp_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "e53fa425d89dfeb0e9395eca9bcd73dc0a55314abf45c5b10b43e2cbfa724345", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.059514161199331284, 0.08157797157764435, 0.14292451739311218, 0.10997853428125381, 0.0991024449467659, 0.1065012738108635, 0.12155740708112717, 0.1095810979604721, 0.06964563578367233, 0.08311934769153595, 0.09363401681184769, 0.0941167026758194, 0.08564268052577972, 0.09493856877088547, 0.09063401818275452, 0.09473779052495956, 0.10505300015211105, 0.09355910867452621, 0.09618847072124481, 0.11416846513748169, 0.047205954790115356, 0.053352247923612595, 0.06512638926506042, 0.07829347252845764, 0.05368548259139061, 0.05310077592730522, 0.061235878616571426, 0.07320034503936768, 0.05304349213838577, 0.0639304369688034, 0.07404901832342148, 0.08157160878181458 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 4.282360038132538e-06, "first_to_last_ratio": 0.16577854373578085, "imbalance_abs_log_ratio": 1.7971024551752253, "last_quartile_mean": 2.58318111718836e-05, "mean": 1.2683608353825093e-05, "normalized": [ 0.7371604731830339, 0.2608676846318322, 0.1772984487091933, 0.31843942285693955, 0.2525707093040399, 0.19973922399636204, 0.2701253989739566, 0.4848344437216343, 0.5588056292473077, 0.42255873596993226, 0.38534811089229865, 0.39272403664612837, 0.4552337220743006, 0.47205763485321933, 0.5295988620451247, 0.5368761557750548, 0.4120249362062748, 0.5137117563009114, 0.4760992890844673, 0.46368316293522016, 1.8372053359532785, 2.0901184966481865, 1.853418275151945, 1.6064639084854364, 1.921337771129799, 2.460138954395665, 2.9276419839576824, 2.160168960627677, 2.4516870755203413, 1.9409071390996004, 1.262941799056306, 1.1682124625668504 ], "peak_layer": 27, "peak_normalized": 2.9276419839576824, "population_cv": 0.8159036704423447, "rest_mean": 1.065961272514768e-05, "spike_contrast": 2.215200998154085, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.3613184748683124e-05, "top_five_layers": [ 27, 26, 29, 28, 22 ], "top_five_spike_overlap": 1 }, "values": [ 9.349854735773988e-06, 3.3087435440393165e-06, 2.248784085168154e-06, 4.0389609239355195e-06, 3.20350795846025e-06, 2.533414090066799e-06, 3.426164767006412e-06, 6.149450200609863e-06, 7.08767174728564e-06, 5.359569513530005e-06, 4.887604518444277e-06, 4.981157871952746e-06, 5.774006240244489e-06, 5.987394160911208e-06, 6.717224550811807e-06, 6.809526894357987e-06, 5.225962922850158e-06, 6.5157187236764e-06, 6.038656920281937e-06, 5.8811756389332e-06, 2.330239294678904e-05, 2.6510244424571283e-05, 2.3508031517849304e-05, 2.0375759049784392e-05, 2.4369495804421604e-05, 3.1203438993543386e-05, 3.713306432473473e-05, 2.739873707469087e-05, 3.1096238672034815e-05, 2.4617706003482454e-05, 1.6018659152905457e-05, 1.4817149349255487e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 1.718563885333424e-05, "first_to_last_ratio": 0.16638017542287623, "imbalance_abs_log_ratio": 1.7934798957762106, "last_quartile_mean": 0.00010329138558518025, "mean": 5.065687940941643e-05, "normalized": [ 0.7449754063400994, 0.2674274841868918, 0.176615953696408, 0.32135055643613986, 0.24800665504407848, 0.19851158311607722, 0.2704757359102896, 0.48668282056560047, 0.5640471659909781, 0.4229737652684744, 0.38647973892545656, 0.3933624848815497, 0.45151620684777444, 0.4688240631089152, 0.5271453320037288, 0.5301233296410177, 0.41139528862260494, 0.5168841060068852, 0.4785636529652508, 0.4612439111418805, 1.8356007727038306, 2.08105235504095, 1.8488854559203878, 1.5955389876194308, 1.917727637778945, 2.458798360210292, 2.9365201131996117, 2.170722638155121, 2.460421978282104, 1.9348898151096774, 1.269562900717559, 1.1636737445619891 ], "peak_layer": 27, "peak_normalized": 2.9365201131996117, "population_cv": 0.8162433373578734, "rest_mean": 4.2629067576382554e-05, "spike_contrast": 2.2052338615982627, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.40070633077994e-05, "top_five_layers": [ 27, 29, 26, 28, 22 ], "top_five_spike_overlap": 1 }, "values": [ 3.773812932195142e-05, 1.3547041817218997e-05, 8.946813068178017e-06, 1.6278616385534406e-05, 1.2563243217300624e-05, 1.0055977327283472e-05, 1.3701456737180706e-05, 2.4653832952026278e-05, 2.857286926882807e-05, 2.142653102055192e-05, 1.95778575289296e-05, 1.9926515960833058e-05, 2.2872402041684836e-05, 2.3749164029140957e-05, 2.670353751454968e-05, 2.6854393581743352e-05, 2.0840001525357366e-05, 2.6183735826634802e-05, 2.4242541257990524e-05, 2.336517718504183e-05, 9.298580698668957e-05, 0.00010541961819399148, 9.365876758238301e-05, 8.082502608885989e-05, 9.714609768707305e-05, 0.00012455505202524364, 0.00014875494525767863, 0.00010996203491231427, 0.00012463729945011437, 9.801548003451899e-05, 6.431209476431832e-05, 5.894808055018075e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.015240396023727953, "first_to_last_ratio": 0.1663801696522023, "imbalance_abs_log_ratio": 1.7934799304598743, "last_quartile_mean": 0.09159983461722732, "mean": 0.04492302719154395, "normalized": [ 0.7449753460489054, 0.26742748904042685, 0.17661593268218315, 0.321350542134168, 0.24800665849213818, 0.1985115882655409, 0.2704757271489617, 0.4866828482518511, 0.5640471952995433, 0.42297377203556435, 0.3864797481073324, 0.3933624878995228, 0.45151618053467946, 0.46882405470538285, 0.5271453409370506, 0.5301232991974691, 0.41139530181327366, 0.5168841099447345, 0.47856363922622325, 0.46124390662687015, 1.8356006679361003, 2.081052228953079, 1.848885590918907, 1.5955389700556024, 1.917727666755993, 2.4587984102120464, 2.93652010100414, 2.1707225152113807, 2.460422102750238, 1.9348898812768835, 1.2695629566079318, 1.1636737399258756 ], "peak_layer": 27, "peak_normalized": 2.93652010100414, "population_cv": 0.8162433411582066, "rest_mean": 0.03780388352633626, "spike_contrast": 2.20523383333323, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.08336640298366546, "top_five_layers": [ 27, 29, 26, 28, 22 ], "top_five_spike_overlap": 1 }, "values": [ 0.03346654772758484, 0.012013652361929417, 0.00793412234634161, 0.01443603914231062, 0.011141209863126278, 0.008917741477489471, 0.012150588445365429, 0.021863266825675964, 0.025338707491755486, 0.01900126226246357, 0.01736184023320675, 0.01767103374004364, 0.020283473655581474, 0.021060995757579803, 0.02368096448481083, 0.023814743384718895, 0.018481122329831123, 0.0232199989259243, 0.021498527377843857, 0.020720472559332848, 0.08246073871850967, 0.09348716586828232, 0.08305753767490387, 0.07167644053697586, 0.08615013211965561, 0.1104566678404808, 0.13191737234592438, 0.09751542657613754, 0.11052960902452469, 0.08692111074924469, 0.05703261122107506, 0.05227574706077576 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 1.0233365017597862e-06, "first_to_last_ratio": 0.1480025981785645, "imbalance_abs_log_ratio": 1.9105254501115427, "last_quartile_mean": 6.914314440109592e-06, "mean": 3.296389566642688e-06, "normalized": [ 0.7608840230855708, 0.24586572262647768, 0.17502887975267875, 0.34247685409924145, 0.22800061621735734, 0.18600379497390226, 0.2149122241264949, 0.3303608075987604, 0.46008975017988785, 0.3860494280463953, 0.3623766661247533, 0.3686778145719148, 0.426652215383595, 0.4827299672018037, 0.5341318312161455, 0.540038054260095, 0.4190297860769054, 0.5176289432662049, 0.46811679609914253, 0.37457141490190266, 1.8268398671368178, 2.082207855022319, 1.8795803251320229, 1.607413091424797, 1.9351478571951781, 2.456862421190237, 3.170929487957598, 2.272255022570422, 2.474770672449538, 1.9284098110964674, 1.3327835317995094, 1.209174467215864 ], "peak_layer": 27, "peak_normalized": 3.170929487957598, "population_cv": 0.8537098015353323, "rest_mean": 2.767601188224944e-06, "spike_contrast": 2.2228082703072234, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.151846810098504e-06, "top_five_layers": [ 27, 29, 26, 28, 22 ], "top_five_spike_overlap": 1 }, "values": [ 2.5081701551243896e-06, 8.104692028609861e-07, 5.769633730778878e-07, 1.1289371286693495e-06, 7.515788524870004e-07, 6.13140969107917e-07, 7.084344133545528e-07, 1.0889979193962063e-06, 1.516635052212223e-06, 1.2725693068205146e-06, 1.1945346614083974e-06, 1.2153057014074875e-06, 1.4064119113754714e-06, 1.5912660273897927e-06, 1.760706595632655e-06, 1.780175807652995e-06, 1.3812854149364284e-06, 1.7063066479749978e-06, 1.543095322631416e-06, 1.2347333040452213e-06, 6.02197587795672e-06, 6.863768248877022e-06, 6.1958289734320715e-06, 5.29865974385757e-06, 6.379001206369139e-06, 8.09877565188799e-06, 1.0452618880663067e-05, 7.490237749152584e-06, 8.157808224495966e-06, 6.356789981509792e-06, 4.393373728817096e-06, 3.985910097981105e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 1.1577771999782271e-05, "first_to_last_ratio": 0.18777331254416324, "imbalance_abs_log_ratio": 1.6725198280345805, "last_quartile_mean": 6.165824015624821e-05, "mean": 3.23950525427108e-05, "normalized": [ 0.8059515428011661, 0.28635608313183425, 0.14375122303573926, 0.30107805378144187, 0.2779058677186346, 0.22065449738835993, 0.2845247997798771, 0.5389236576597887, 0.6310294503025052, 0.4689064640888303, 0.42743449943314926, 0.44534953670243166, 0.5186946978022778, 0.5244663791583728, 0.5831250565491412, 0.5813066315981038, 0.45997796108159217, 0.5838076748023374, 0.5290689567347687, 0.5139009287187557, 1.9512860349727317, 2.203113268818907, 1.9318335765401984, 1.560971240310136, 1.9741434428890046, 2.418318204639279, 2.7522390849796614, 2.0429800218846808, 2.1454782099817096, 1.7273762424807697, 1.129858255585034, 1.0361884546487803 ], "peak_layer": 27, "peak_normalized": 2.7522390849796614, "population_cv": 0.7689049336052842, "rest_mean": 2.6850282278313966e-05, "spike_contrast": 2.3216445668729886, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.23368119704537e-05, "top_five_layers": [ 27, 26, 22, 29, 28 ], "top_five_spike_overlap": 1 }, "values": [ 2.610884257592261e-05, 9.276520358980633e-06, 4.656828423321713e-06, 9.753439371706918e-06, 9.002775186672807e-06, 7.148114036681363e-06, 9.217195838573389e-06, 1.745846020639874e-05, 2.0442232198547572e-05, 1.5190249541774392e-05, 1.384676306770416e-05, 1.4427121641347185e-05, 1.6803141988930292e-05, 1.699011590972077e-05, 1.8890366845880635e-05, 1.8831458874046803e-05, 1.4901010217727162e-05, 1.8912480300059542e-05, 1.713921665214002e-05, 1.664784758759197e-05, 6.321201362879947e-05, 7.136997010093182e-05, 6.258185021579266e-05, 5.05677453475073e-05, 6.39524805592373e-05, 7.83415453042835e-05, 8.915892976801842e-05, 6.61824451526627e-05, 6.95028793415986e-05, 5.5958444136194885e-05, 3.6601817555492744e-05, 3.3567379432497546e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 7.537063339668748e-06, "first_to_last_ratio": 0.2196410574616343, "imbalance_abs_log_ratio": 1.5157606220609539, "last_quartile_mean": 3.431536629250331e-05, "mean": 1.99045535751452e-05, "normalized": [ 0.8516144687475854, 0.30329532191655734, 0.12972062496162942, 0.2856119129431854, 0.3157110369274372, 0.24651089695341166, 0.3041308146875524, 0.592686965773776, 0.6958324208175611, 0.507473054430893, 0.4520439463300317, 0.4869439209822031, 0.5862552935313644, 0.5830348652841818, 0.6383685668274365, 0.6346090549755397, 0.5196259673267944, 0.671415881381562, 0.5884992667325439, 0.5831102583972331, 2.1793369033416923, 2.443723827249123, 2.0979779561011083, 1.5105005630535813, 1.9779981784616936, 2.3612530829014102, 2.448898079444227, 1.7971309227220995, 1.817356199078795, 1.5022550238967485, 0.9745901402356775, 0.912484583585365 ], "peak_layer": 27, "peak_normalized": 2.448898079444227, "population_cv": 0.7302664213607039, "rest_mean": 1.606405295138689e-05, "spike_contrast": 2.530074886234182, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.0643256943440066e-05, "top_five_layers": [ 27, 22, 26, 21, 23 ], "top_five_spike_overlap": 3 }, "values": [ 1.695100581855513e-05, 6.0369579841790255e-06, 2.582031129350071e-06, 5.684977622877341e-06, 6.284087248786818e-06, 4.9066893552662805e-06, 6.0535880948009435e-06, 1.1797169463534374e-05, 1.3850233699486125e-05, 1.0101024599862285e-05, 8.997732948046178e-06, 9.692401363281533e-06, 1.1669149898807518e-05, 1.1605048712226562e-05, 1.2706441339105368e-05, 1.2631609934032895e-05, 1.034292290569283e-05, 1.3364233382162638e-05, 1.1713815183611587e-05, 1.1606549378484488e-05, 4.337872815085575e-05, 4.864123184233904e-05, 4.175931462668814e-05, 3.0065839382587e-05, 3.93711707147304e-05, 4.699968849308789e-05, 4.8744223022367805e-05, 3.577108873287216e-05, 3.617366382968612e-05, 2.9901715606683865e-05, 1.9398781660129316e-05, 1.8162598280468956e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 3.494163581763132e-05, "first_to_last_ratio": 0.15651626375472188, "imbalance_abs_log_ratio": 1.8545953526460093, "last_quartile_mean": 0.00022324603833112633, "mean": 0.00010677687006932501, "normalized": [ 0.7321307956101275, 0.26258629565235714, 0.14084635033633347, 0.29511142007599656, 0.24288419587697632, 0.1988436860580881, 0.27076714984615463, 0.4747480728559904, 0.5774219445213173, 0.4369513360451195, 0.397280429874437, 0.4031380268242667, 0.444174220127482, 0.4777542803038818, 0.5358453534538501, 0.5305966707491948, 0.4246608899008583, 0.5233998849385313, 0.48776848697825403, 0.4595022529732046, 1.6899286448236293, 1.9477123814914694, 1.7666943601373737, 1.5530807558402215, 2.020597563184557, 2.6592203636901113, 3.2036592034115787, 2.3671756608929946, 2.34022861615937, 1.837074516597572, 1.1950438133134078, 1.1031723774552928 ], "peak_layer": 27, "peak_normalized": 3.2036592034115787, "population_cv": 0.8394926512497429, "rest_mean": 9.104502367807543e-05, "spike_contrast": 2.1058684246159736, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00019172884058207273, "top_five_layers": [ 27, 26, 28, 29, 25 ], "top_five_spike_overlap": 1 }, "values": [ 7.817463483661413e-05, 2.80381427728571e-05, 1.5039132449601311e-05, 3.1511073757428676e-05, 2.5934414225048386e-05, 2.1231906430330127e-05, 2.8911668778164312e-05, 5.0692113291006535e-05, 6.16553079453297e-05, 4.6656296035507694e-05, 4.242036084178835e-05, 4.304581671021879e-05, 4.742753299069591e-05, 5.1013106713071465e-05, 5.721588968299329e-05, 5.66554517718032e-05, 4.5343960664467886e-05, 5.588700150838122e-05, 5.208239235798828e-05, 4.906421236228198e-05, 0.00018044529133476317, 0.00020797063189093024, 0.00018864209414459765, 0.00016583310207352042, 0.0002157530834665522, 0.0002839432272594422, 0.0003420767025090754, 0.00025275960797443986, 0.00024988228688016534, 0.000196157066966407, 0.00012760303798131645, 0.00011779329361161217 ] } } } }, "selector": { "expected_uniform_count": 8, "identities_unique": true, "identity_order_sha256": "97fd9721a263024f6a746638f32e0ad9a988cd4c111c00e9e178c1022e5c9d5e", "passed": true, "uniform_count": 8, "uniform_indices": [ 48, 49, 50, 51, 52, 53, 54, 55 ], "visit_count": 65, "visits": [ { "branch": "attention", "coefficient": "detached_learned", "group": 1, "index": 0, "kind": "depth", "layer": 1, "offset": 1, "sources": 1, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 1, "index": 1, "kind": "depth", "layer": 1, "offset": 1, "sources": 2, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 1, "index": 2, "kind": "depth", "layer": 2, "offset": 2, "sources": 2, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 1, "index": 3, "kind": "depth", "layer": 2, "offset": 2, "sources": 2, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 1, "index": 4, "kind": "depth", "layer": 3, "offset": 3, "sources": 2, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 1, "index": 5, "kind": "depth", "layer": 3, "offset": 3, "sources": 2, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 1, "index": 6, "kind": "depth", "layer": 4, "offset": 4, "sources": 2, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 1, "index": 7, "kind": "depth", "layer": 4, "offset": 4, "sources": 2, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 2, "index": 8, "kind": "depth", "layer": 5, "offset": 1, "sources": 2, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 2, "index": 9, "kind": "depth", "layer": 5, "offset": 1, "sources": 3, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 2, "index": 10, "kind": "depth", "layer": 6, "offset": 2, "sources": 3, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 2, "index": 11, "kind": "depth", "layer": 6, "offset": 2, "sources": 3, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 2, "index": 12, "kind": "depth", "layer": 7, "offset": 3, "sources": 3, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 2, "index": 13, "kind": "depth", "layer": 7, "offset": 3, "sources": 3, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 2, "index": 14, "kind": "depth", "layer": 8, "offset": 4, "sources": 3, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 2, "index": 15, "kind": "depth", "layer": 8, "offset": 4, "sources": 3, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 3, "index": 16, "kind": "depth", "layer": 9, "offset": 1, "sources": 3, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 3, "index": 17, "kind": "depth", "layer": 9, "offset": 1, "sources": 4, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 3, "index": 18, "kind": "depth", "layer": 10, "offset": 2, "sources": 4, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 3, "index": 19, "kind": "depth", "layer": 10, "offset": 2, "sources": 4, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 3, "index": 20, "kind": "depth", "layer": 11, "offset": 3, "sources": 4, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 3, "index": 21, "kind": "depth", "layer": 11, "offset": 3, "sources": 4, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 3, "index": 22, "kind": "depth", "layer": 12, "offset": 4, "sources": 4, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 3, "index": 23, "kind": "depth", "layer": 12, "offset": 4, "sources": 4, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 4, "index": 24, "kind": "depth", "layer": 13, "offset": 1, "sources": 4, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 4, "index": 25, "kind": "depth", "layer": 13, "offset": 1, "sources": 5, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 4, "index": 26, "kind": "depth", "layer": 14, "offset": 2, "sources": 5, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 4, "index": 27, "kind": "depth", "layer": 14, "offset": 2, "sources": 5, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 4, "index": 28, "kind": "depth", "layer": 15, "offset": 3, "sources": 5, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 4, "index": 29, "kind": "depth", "layer": 15, "offset": 3, "sources": 5, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 4, "index": 30, "kind": "depth", "layer": 16, "offset": 4, "sources": 5, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 4, "index": 31, "kind": "depth", "layer": 16, "offset": 4, "sources": 5, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 5, "index": 32, "kind": "depth", "layer": 17, "offset": 1, "sources": 5, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 5, "index": 33, "kind": "depth", "layer": 17, "offset": 1, "sources": 6, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 5, "index": 34, "kind": "depth", "layer": 18, "offset": 2, "sources": 6, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 5, "index": 35, "kind": "depth", "layer": 18, "offset": 2, "sources": 6, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 5, "index": 36, "kind": "depth", "layer": 19, "offset": 3, "sources": 6, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 5, "index": 37, "kind": "depth", "layer": 19, "offset": 3, "sources": 6, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 5, "index": 38, "kind": "depth", "layer": 20, "offset": 4, "sources": 6, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 5, "index": 39, "kind": "depth", "layer": 20, "offset": 4, "sources": 6, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 6, "index": 40, "kind": "depth", "layer": 21, "offset": 1, "sources": 6, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 6, "index": 41, "kind": "depth", "layer": 21, "offset": 1, "sources": 7, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 6, "index": 42, "kind": "depth", "layer": 22, "offset": 2, "sources": 7, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 6, "index": 43, "kind": "depth", "layer": 22, "offset": 2, "sources": 7, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 6, "index": 44, "kind": "depth", "layer": 23, "offset": 3, "sources": 7, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 6, "index": 45, "kind": "depth", "layer": 23, "offset": 3, "sources": 7, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 6, "index": 46, "kind": "depth", "layer": 24, "offset": 4, "sources": 7, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 6, "index": 47, "kind": "depth", "layer": 24, "offset": 4, "sources": 7, "use_uniform": false }, { "branch": "attention", "coefficient": "uniform", "group": 7, "index": 48, "kind": "depth", "layer": 25, "offset": 1, "sources": 7, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 7, "index": 49, "kind": "depth", "layer": 25, "offset": 1, "sources": 8, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 7, "index": 50, "kind": "depth", "layer": 26, "offset": 2, "sources": 8, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 7, "index": 51, "kind": "depth", "layer": 26, "offset": 2, "sources": 8, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 7, "index": 52, "kind": "depth", "layer": 27, "offset": 3, "sources": 8, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 7, "index": 53, "kind": "depth", "layer": 27, "offset": 3, "sources": 8, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 7, "index": 54, "kind": "depth", "layer": 28, "offset": 4, "sources": 8, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 7, "index": 55, "kind": "depth", "layer": 28, "offset": 4, "sources": 8, "use_uniform": true }, { "branch": "attention", "coefficient": "detached_learned", "group": 8, "index": 56, "kind": "depth", "layer": 29, "offset": 1, "sources": 8, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 8, "index": 57, "kind": "depth", "layer": 29, "offset": 1, "sources": 9, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 8, "index": 58, "kind": "depth", "layer": 30, "offset": 2, "sources": 9, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 8, "index": 59, "kind": "depth", "layer": 30, "offset": 2, "sources": 9, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 8, "index": 60, "kind": "depth", "layer": 31, "offset": 3, "sources": 9, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 8, "index": 61, "kind": "depth", "layer": 31, "offset": 3, "sources": 9, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 8, "index": 62, "kind": "depth", "layer": 32, "offset": 4, "sources": 9, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 8, "index": 63, "kind": "depth", "layer": 32, "offset": 4, "sources": 9, "use_uniform": false }, { "branch": "output", "coefficient": "detached_learned", "group": null, "index": 64, "kind": "output", "layer": null, "offset": null, "sources": 9, "use_uniform": false } ], "visits_sha256": "c9df79ea46ffe906b779be180e1a1ca10bc72a061b713a1b39e60845020b9b00" } }, "uniform_groups_6_7_only": { "bits_per_byte": 1.8740129107275643, "forward": { "activation_sha256": { "attention_branch_output": "5e6ecbcd78543fc42d4098a51169dca446d404ce384015643e84d2c78433c949", "mlp_branch_output": "c0b8018c349b1bd09e5d17858eb312a2323da0e92de9651e2bb08ae4adf115c7", "post_attention_state": "90a633f12d984935028fe5d7212be48e7da7b4a0806751939a551938d16dc540", "post_mlp_state": "89f2226456bfe96e245d2aad0f699d32c676f9f53099789cd870253d002402f5", "pre_attention_input": "ee695e78937de01b7128d3866b8fc5dd328ebdf2c159be37918d4f02952d5248", "pre_mlp_input": "e53fa425d89dfeb0e9395eca9bcd73dc0a55314abf45c5b10b43e2cbfa724345" }, "logits_sha256": "bdacba5675f0579cec629bbf01b36c8660ad576cfdf9fef1bea8259eaeac9ec2", "loss_nats": 1.2989667654037476, "mixer_summary_sha256": "eb3ac0f07de9b755d6e67a510ab68d09d9ad666bb7b5d020ba7d8c5055c399b2" }, "loss_nats": 1.2989667654037476, "loss_scale": 1.0, "mixers": [ { "branch": "attention", "entropy_mean": -0.0, "group": 1, "latest_source_mass": 1.0, "layer": 1, "max_source_mass": 1.0, "mixer_index": 0, "normalized_entropy": 1.0, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 1.0, "median": 1.0, "p05": 1.0, "p95": 1.0 } ], "sources": 1 }, { "branch": "mlp", "entropy_mean": 0.22886337339878082, "group": 1, "latest_source_mass": 0.06350618600845337, "layer": 1, "max_source_mass": 0.9364938139915466, "mixer_index": 1, "normalized_entropy": 0.3301800538435402, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.9364938139915466, "median": 0.9452826976776123, "p05": 0.8725647330284119, "p95": 0.9712384939193726 }, { "label": "current_group_partial", "mean": 0.06350618600845337, "median": 0.0547172985970974, "p05": 0.028761589899659157, "p95": 0.12743529677391052 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.603729784488678, "group": 1, "latest_source_mass": 0.4133155345916748, "layer": 2, "max_source_mass": 0.5866844654083252, "mixer_index": 2, "normalized_entropy": 0.8709979661187783, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.5866844654083252, "median": 0.5734395384788513, "p05": 0.29813364148139954, "p95": 0.8843642473220825 }, { "label": "current_group_partial", "mean": 0.4133155345916748, "median": 0.4265604615211487, "p05": 0.11563576757907867, "p95": 0.7018663287162781 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.3967982828617096, "group": 1, "latest_source_mass": 0.14110636711120605, "layer": 2, "max_source_mass": 0.858893632888794, "mixer_index": 3, "normalized_entropy": 0.5724589149178446, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.858893632888794, "median": 0.8705973029136658, "p05": 0.7599387168884277, "p95": 0.9270997643470764 }, { "label": "current_group_partial", "mean": 0.14110636711120605, "median": 0.12940272688865662, "p05": 0.07290027290582657, "p95": 0.2400612235069275 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6852496862411499, "group": 1, "latest_source_mass": 0.518975019454956, "layer": 3, "max_source_mass": 0.518975019454956, "mixer_index": 4, "normalized_entropy": 0.9886063241108252, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.48102498054504395, "median": 0.48102718591690063, "p05": 0.38786420226097107, "p95": 0.580143928527832 }, { "label": "current_group_partial", "mean": 0.518975019454956, "median": 0.5189728140830994, "p05": 0.41985607147216797, "p95": 0.6121357679367065 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.5890734195709229, "group": 1, "latest_source_mass": 0.3123423159122467, "layer": 3, "max_source_mass": 0.6876576542854309, "mixer_index": 5, "normalized_entropy": 0.8498533011344741, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.6876576542854309, "median": 0.6909762620925903, "p05": 0.48803940415382385, "p95": 0.8900076746940613 }, { "label": "current_group_partial", "mean": 0.3123423159122467, "median": 0.3090237081050873, "p05": 0.10999235510826111, "p95": 0.5119606256484985 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6873323321342468, "group": 1, "latest_source_mass": 0.4831485152244568, "layer": 4, "max_source_mass": 0.5168514251708984, "mixer_index": 6, "normalized_entropy": 0.9916109470127238, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.5168514251708984, "median": 0.5219584703445435, "p05": 0.4251725673675537, "p95": 0.5947866439819336 }, { "label": "current_group_partial", "mean": 0.4831485152244568, "median": 0.47804147005081177, "p05": 0.405213326215744, "p95": 0.5748274326324463 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.487301766872406, "group": 1, "latest_source_mass": 0.2060384750366211, "layer": 4, "max_source_mass": 0.7939615249633789, "mixer_index": 7, "normalized_entropy": 0.7030278424832499, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.7939615249633789, "median": 0.7953404188156128, "p05": 0.6590759754180908, "p95": 0.9329437017440796 }, { "label": "current_group_partial", "mean": 0.2060384750366211, "median": 0.20465964078903198, "p05": 0.06705628335475922, "p95": 0.3409239649772644 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.674697756767273, "group": 2, "latest_source_mass": 0.42952626943588257, "layer": 5, "max_source_mass": 0.5704737305641174, "mixer_index": 8, "normalized_entropy": 0.9733831077870527, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.5704737305641174, "median": 0.5696132779121399, "p05": 0.4695202708244324, "p95": 0.6750621199607849 }, { "label": "completed_group_1", "mean": 0.42952626943588257, "median": 0.4303867816925049, "p05": 0.3249378800392151, "p95": 0.5304797887802124 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.9691112041473389, "group": 2, "latest_source_mass": 0.2681158185005188, "layer": 5, "max_source_mass": 0.5506307482719421, "mixer_index": 9, "normalized_entropy": 0.8821230329784768, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.5506307482719421, "median": 0.543993353843689, "p05": 0.41262108087539673, "p95": 0.7161091566085815 }, { "label": "completed_group_1", "mean": 0.18125343322753906, "median": 0.1794438362121582, "p05": 0.11575750261545181, "p95": 0.25031501054763794 }, { "label": "current_group_partial", "mean": 0.2681158185005188, "median": 0.2664111256599426, "p05": 0.1564716249704361, "p95": 0.3817881643772125 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0745112895965576, "group": 2, "latest_source_mass": 0.412794291973114, "layer": 6, "max_source_mass": 0.412794291973114, "mixer_index": 10, "normalized_entropy": 0.9780623252441762, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.34262996912002563, "median": 0.3412924110889435, "p05": 0.29556700587272644, "p95": 0.39125555753707886 }, { "label": "completed_group_1", "mean": 0.24457570910453796, "median": 0.24382859468460083, "p05": 0.22120657563209534, "p95": 0.2734297513961792 }, { "label": "current_group_partial", "mean": 0.412794291973114, "median": 0.4151499271392822, "p05": 0.37735897302627563, "p95": 0.43749260902404785 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0074045658111572, "group": 2, "latest_source_mass": 0.3775783181190491, "layer": 6, "max_source_mass": 0.4399106502532959, "mixer_index": 11, "normalized_entropy": 0.9169791528842925, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.4399106502532959, "median": 0.44856905937194824, "p05": 0.22063353657722473, "p95": 0.6359403133392334 }, { "label": "completed_group_1", "mean": 0.18251106142997742, "median": 0.18562838435173035, "p05": 0.1196163073182106, "p95": 0.23045453429222107 }, { "label": "current_group_partial", "mean": 0.3775783181190491, "median": 0.3675365447998047, "p05": 0.23817461729049683, "p95": 0.5557451844215393 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.060497522354126, "group": 2, "latest_source_mass": 0.4025024175643921, "layer": 7, "max_source_mass": 0.4025024175643921, "mixer_index": 12, "normalized_entropy": 0.9653064445872968, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.3786153197288513, "median": 0.3907395303249359, "p05": 0.28039106726646423, "p95": 0.4436405599117279 }, { "label": "completed_group_1", "mean": 0.2188822478055954, "median": 0.21226292848587036, "p05": 0.1885872334241867, "p95": 0.2672030031681061 }, { "label": "current_group_partial", "mean": 0.4025024175643921, "median": 0.39815032482147217, "p05": 0.3533596992492676, "p95": 0.460100919008255 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 0.9941019415855408, "group": 2, "latest_source_mass": 0.4801652431488037, "layer": 7, "max_source_mass": 0.4801652431488037, "mixer_index": 13, "normalized_entropy": 0.9048705824970601, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.32296955585479736, "median": 0.2803739905357361, "p05": 0.1784922033548355, "p95": 0.6440245509147644 }, { "label": "completed_group_1", "mean": 0.19686520099639893, "median": 0.2053825855255127, "p05": 0.09813719242811203, "p95": 0.2504696547985077 }, { "label": "current_group_partial", "mean": 0.4801652431488037, "median": 0.5057140588760376, "p05": 0.2529003322124481, "p95": 0.598870575428009 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0503522157669067, "group": 2, "latest_source_mass": 0.4001518487930298, "layer": 8, "max_source_mass": 0.40130966901779175, "mixer_index": 14, "normalized_entropy": 0.9560717885654542, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.40130966901779175, "median": 0.396759569644928, "p05": 0.3479611873626709, "p95": 0.4733521044254303 }, { "label": "completed_group_1", "mean": 0.19853851199150085, "median": 0.19769755005836487, "p05": 0.17136667668819427, "p95": 0.22788912057876587 }, { "label": "current_group_partial", "mean": 0.4001518487930298, "median": 0.40291696786880493, "p05": 0.3506356179714203, "p95": 0.43941178917884827 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0129402875900269, "group": 2, "latest_source_mass": 0.42161279916763306, "layer": 8, "max_source_mass": 0.42161279916763306, "mixer_index": 15, "normalized_entropy": 0.9220179839951123, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.4093267023563385, "median": 0.4046899676322937, "p05": 0.2895161807537079, "p95": 0.5762131810188293 }, { "label": "completed_group_1", "mean": 0.16906049847602844, "median": 0.16888955235481262, "p05": 0.11233551055192947, "p95": 0.22477403283119202 }, { "label": "current_group_partial", "mean": 0.42161279916763306, "median": 0.4227708578109741, "p05": 0.3131049573421478, "p95": 0.519075334072113 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0774019956588745, "group": 3, "latest_source_mass": 0.3560434877872467, "layer": 9, "max_source_mass": 0.38510820269584656, "mixer_index": 16, "normalized_entropy": 0.9806935592947451, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.38510820269584656, "median": 0.3892287313938141, "p05": 0.2934730052947998, "p95": 0.4718902111053467 }, { "label": "completed_group_1", "mean": 0.25884830951690674, "median": 0.25734007358551025, "p05": 0.22217319905757904, "p95": 0.29551801085472107 }, { "label": "completed_group_2", "mean": 0.3560434877872467, "median": 0.35412120819091797, "p05": 0.29444968700408936, "p95": 0.41908568143844604 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.2740377187728882, "group": 3, "latest_source_mass": 0.3633861243724823, "layer": 9, "max_source_mass": 0.3633861243724823, "mixer_index": 17, "normalized_entropy": 0.9190239493895668, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.29550763964653015, "median": 0.30360478162765503, "p05": 0.16904757916927338, "p95": 0.45146965980529785 }, { "label": "completed_group_1", "mean": 0.09541244804859161, "median": 0.09486871212720871, "p05": 0.061360105872154236, "p95": 0.13037073612213135 }, { "label": "completed_group_2", "mean": 0.24569380283355713, "median": 0.24333015084266663, "p05": 0.1698218435049057, "p95": 0.31589236855506897 }, { "label": "current_group_partial", "mean": 0.3633861243724823, "median": 0.36472851037979126, "p05": 0.2795379161834717, "p95": 0.4399876892566681 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.360302448272705, "group": 3, "latest_source_mass": 0.28569501638412476, "layer": 10, "max_source_mass": 0.3109527826309204, "mixer_index": 18, "normalized_entropy": 0.9812507981160736, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.3109527826309204, "median": 0.32172977924346924, "p05": 0.23520725965499878, "p95": 0.37052470445632935 }, { "label": "completed_group_1", "mean": 0.20984819531440735, "median": 0.20958667993545532, "p05": 0.1891367882490158, "p95": 0.23184512555599213 }, { "label": "completed_group_2", "mean": 0.19350400567054749, "median": 0.19153150916099548, "p05": 0.16607457399368286, "p95": 0.22877082228660583 }, { "label": "current_group_partial", "mean": 0.28569501638412476, "median": 0.28060320019721985, "p05": 0.23824527859687805, "p95": 0.3439165949821472 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.2722704410552979, "group": 3, "latest_source_mass": 0.29912811517715454, "layer": 10, "max_source_mass": 0.35036465525627136, "mixer_index": 19, "normalized_entropy": 0.9177491279900463, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.35036465525627136, "median": 0.36210548877716064, "p05": 0.12807878851890564, "p95": 0.5797606706619263 }, { "label": "completed_group_1", "mean": 0.12631990015506744, "median": 0.1264108121395111, "p05": 0.06977735459804535, "p95": 0.1755298376083374 }, { "label": "completed_group_2", "mean": 0.22418735921382904, "median": 0.22563719749450684, "p05": 0.1176326721906662, "p95": 0.32001155614852905 }, { "label": "current_group_partial", "mean": 0.29912811517715454, "median": 0.28589946031570435, "p05": 0.193109393119812, "p95": 0.44119665026664734 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3558036088943481, "group": 3, "latest_source_mass": 0.314197838306427, "layer": 11, "max_source_mass": 0.314197838306427, "mixer_index": 20, "normalized_entropy": 0.978005571485618, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.298976868391037, "median": 0.2961939573287964, "p05": 0.24950163066387177, "p95": 0.35835936665534973 }, { "label": "completed_group_1", "mean": 0.1941717267036438, "median": 0.1918032318353653, "p05": 0.16809208691120148, "p95": 0.22895419597625732 }, { "label": "completed_group_2", "mean": 0.1926535815000534, "median": 0.19306811690330505, "p05": 0.16983944177627563, "p95": 0.2155935913324356 }, { "label": "current_group_partial", "mean": 0.314197838306427, "median": 0.31541913747787476, "p05": 0.2673022747039795, "p95": 0.35941022634506226 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.2696614265441895, "group": 3, "latest_source_mass": 0.2862822115421295, "layer": 11, "max_source_mass": 0.3667730689048767, "mixer_index": 21, "normalized_entropy": 0.9158671218416545, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.3667730689048767, "median": 0.4067659378051758, "p05": 0.10593366622924805, "p95": 0.5672274827957153 }, { "label": "completed_group_1", "mean": 0.15310388803482056, "median": 0.15050218999385834, "p05": 0.08651046454906464, "p95": 0.22459912300109863 }, { "label": "completed_group_2", "mean": 0.19384080171585083, "median": 0.18732509016990662, "p05": 0.1267009675502777, "p95": 0.281107634305954 }, { "label": "current_group_partial", "mean": 0.2862822115421295, "median": 0.26271265745162964, "p05": 0.17024393379688263, "p95": 0.475044310092926 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3606936931610107, "group": 3, "latest_source_mass": 0.3028184771537781, "layer": 12, "max_source_mass": 0.3028184771537781, "mixer_index": 22, "normalized_entropy": 0.9815330216461395, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.298227459192276, "median": 0.2954302132129669, "p05": 0.25287362933158875, "p95": 0.35405808687210083 }, { "label": "completed_group_1", "mean": 0.18611131608486176, "median": 0.1850002110004425, "p05": 0.15901757776737213, "p95": 0.21725836396217346 }, { "label": "completed_group_2", "mean": 0.21284276247024536, "median": 0.21402311325073242, "p05": 0.18819917738437653, "p95": 0.23418807983398438 }, { "label": "current_group_partial", "mean": 0.3028184771537781, "median": 0.3041122555732727, "p05": 0.25661808252334595, "p95": 0.34458184242248535 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.269316554069519, "group": 3, "latest_source_mass": 0.2653314769268036, "layer": 12, "max_source_mass": 0.4081852436065674, "mixer_index": 23, "normalized_entropy": 0.9156183489371815, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.4081852436065674, "median": 0.4254342317581177, "p05": 0.20765496790409088, "p95": 0.587238073348999 }, { "label": "completed_group_1", "mean": 0.14420127868652344, "median": 0.1438048928976059, "p05": 0.1064920499920845, "p95": 0.1871720254421234 }, { "label": "completed_group_2", "mean": 0.18228203058242798, "median": 0.17507661879062653, "p05": 0.12183455377817154, "p95": 0.26505547761917114 }, { "label": "current_group_partial", "mean": 0.2653314769268036, "median": 0.24924281239509583, "p05": 0.15609969198703766, "p95": 0.4061311185359955 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3636136054992676, "group": 4, "latest_source_mass": 0.3136237859725952, "layer": 13, "max_source_mass": 0.3136237859725952, "mixer_index": 24, "normalized_entropy": 0.9836392931712563, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.27427661418914795, "median": 0.27403637766838074, "p05": 0.22928935289382935, "p95": 0.319773405790329 }, { "label": "completed_group_1", "mean": 0.21341991424560547, "median": 0.20697325468063354, "p05": 0.18552586436271667, "p95": 0.26688331365585327 }, { "label": "completed_group_2", "mean": 0.19867971539497375, "median": 0.197545126080513, "p05": 0.1794913411140442, "p95": 0.222100168466568 }, { "label": "completed_group_3", "mean": 0.3136237859725952, "median": 0.31875866651535034, "p05": 0.24156072735786438, "p95": 0.3695686459541321 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.570773959159851, "group": 4, "latest_source_mass": 0.2294243425130844, "layer": 13, "max_source_mass": 0.2658740282058716, "mixer_index": 25, "normalized_entropy": 0.9759767351225285, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.2658740282058716, "median": 0.2747163474559784, "p05": 0.17927880585193634, "p95": 0.3512483835220337 }, { "label": "completed_group_1", "mean": 0.13419361412525177, "median": 0.13374894857406616, "p05": 0.11072897166013718, "p95": 0.1586270034313202 }, { "label": "completed_group_2", "mean": 0.1907704919576645, "median": 0.19074805080890656, "p05": 0.14699944853782654, "p95": 0.23316305875778198 }, { "label": "completed_group_3", "mean": 0.17973749339580536, "median": 0.17616114020347595, "p05": 0.1363743096590042, "p95": 0.23202598094940186 }, { "label": "current_group_partial", "mean": 0.2294243425130844, "median": 0.22995705902576447, "p05": 0.17975632846355438, "p95": 0.2789157032966614 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.5560038089752197, "group": 4, "latest_source_mass": 0.32468360662460327, "layer": 14, "max_source_mass": 0.32468360662460327, "mixer_index": 26, "normalized_entropy": 0.9667995248241249, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.20477107167243958, "median": 0.206403911113739, "p05": 0.16619175672531128, "p95": 0.2431035190820694 }, { "label": "completed_group_1", "mean": 0.15721669793128967, "median": 0.15658551454544067, "p05": 0.13878680765628815, "p95": 0.17778633534908295 }, { "label": "completed_group_2", "mean": 0.13811248540878296, "median": 0.13743364810943604, "p05": 0.12352626770734787, "p95": 0.15553249418735504 }, { "label": "completed_group_3", "mean": 0.1752161681652069, "median": 0.17309701442718506, "p05": 0.15223677456378937, "p95": 0.20474471151828766 }, { "label": "current_group_partial", "mean": 0.32468360662460327, "median": 0.32497870922088623, "p05": 0.2946181893348694, "p95": 0.35203880071640015 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.5659973621368408, "group": 4, "latest_source_mass": 0.26703086495399475, "layer": 14, "max_source_mass": 0.26703086495399475, "mixer_index": 27, "normalized_entropy": 0.9730088685238187, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.20531430840492249, "median": 0.20229148864746094, "p05": 0.10698921978473663, "p95": 0.3393132984638214 }, { "label": "completed_group_1", "mean": 0.15986156463623047, "median": 0.15952220559120178, "p05": 0.11969029903411865, "p95": 0.20180393755435944 }, { "label": "completed_group_2", "mean": 0.21557100117206573, "median": 0.21654027700424194, "p05": 0.1673085242509842, "p95": 0.2610440254211426 }, { "label": "completed_group_3", "mean": 0.15222224593162537, "median": 0.1510905921459198, "p05": 0.11800260841846466, "p95": 0.18893754482269287 }, { "label": "current_group_partial", "mean": 0.26703086495399475, "median": 0.258539617061615, "p05": 0.19330690801143646, "p95": 0.36521556973457336 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.5552496910095215, "group": 4, "latest_source_mass": 0.31774693727493286, "layer": 15, "max_source_mass": 0.31774693727493286, "mixer_index": 28, "normalized_entropy": 0.9663309649872576, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.2162160575389862, "median": 0.2145705670118332, "p05": 0.17727568745613098, "p95": 0.26878347992897034 }, { "label": "completed_group_1", "mean": 0.17982497811317444, "median": 0.18024154007434845, "p05": 0.15453973412513733, "p95": 0.2030555009841919 }, { "label": "completed_group_2", "mean": 0.1307673156261444, "median": 0.13107559084892273, "p05": 0.11463287472724915, "p95": 0.14630235731601715 }, { "label": "completed_group_3", "mean": 0.1554446816444397, "median": 0.15516987442970276, "p05": 0.13627474009990692, "p95": 0.1754152774810791 }, { "label": "current_group_partial", "mean": 0.31774693727493286, "median": 0.319039523601532, "p05": 0.27606821060180664, "p95": 0.35650834441185 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.5068919658660889, "group": 4, "latest_source_mass": 0.3298327624797821, "layer": 15, "max_source_mass": 0.3298327624797821, "mixer_index": 29, "normalized_entropy": 0.9362846209998111, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.20646697282791138, "median": 0.18775638937950134, "p05": 0.0806753933429718, "p95": 0.4274584949016571 }, { "label": "completed_group_1", "mean": 0.1197461187839508, "median": 0.11989086866378784, "p05": 0.0809398666024208, "p95": 0.15743419528007507 }, { "label": "completed_group_2", "mean": 0.19215428829193115, "median": 0.19620640575885773, "p05": 0.1148109883069992, "p95": 0.2491212636232376 }, { "label": "completed_group_3", "mean": 0.15179988741874695, "median": 0.15212899446487427, "p05": 0.10697861760854721, "p95": 0.19542087614536285 }, { "label": "current_group_partial", "mean": 0.3298327624797821, "median": 0.32837340235710144, "p05": 0.22746729850769043, "p95": 0.4339301884174347 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.5435223579406738, "group": 4, "latest_source_mass": 0.3224271535873413, "layer": 16, "max_source_mass": 0.3224271535873413, "mixer_index": 30, "normalized_entropy": 0.9590443632623663, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.22410975396633148, "median": 0.22760415077209473, "p05": 0.14593973755836487, "p95": 0.30093055963516235 }, { "label": "completed_group_1", "mean": 0.16986176371574402, "median": 0.16963240504264832, "p05": 0.13569067418575287, "p95": 0.20651353895664215 }, { "label": "completed_group_2", "mean": 0.13080403208732605, "median": 0.13075244426727295, "p05": 0.10945598781108856, "p95": 0.15187400579452515 }, { "label": "completed_group_3", "mean": 0.15279728174209595, "median": 0.15036612749099731, "p05": 0.12810495495796204, "p95": 0.18468116223812103 }, { "label": "current_group_partial", "mean": 0.3224271535873413, "median": 0.3164973258972168, "p05": 0.2600978910923004, "p95": 0.40530598163604736 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.4949920177459717, "group": 4, "latest_source_mass": 0.31847020983695984, "layer": 16, "max_source_mass": 0.31847020983695984, "mixer_index": 31, "normalized_entropy": 0.9288907675133354, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.27174079418182373, "median": 0.2585451304912567, "p05": 0.13639721274375916, "p95": 0.4698348045349121 }, { "label": "completed_group_1", "mean": 0.12178683280944824, "median": 0.11714950203895569, "p05": 0.08594125509262085, "p95": 0.1723237782716751 }, { "label": "completed_group_2", "mean": 0.1529826819896698, "median": 0.15223106741905212, "p05": 0.10371473431587219, "p95": 0.203575000166893 }, { "label": "completed_group_3", "mean": 0.1350194811820984, "median": 0.13363957405090332, "p05": 0.08964549005031586, "p95": 0.18468207120895386 }, { "label": "current_group_partial", "mean": 0.31847020983695984, "median": 0.3185216188430786, "p05": 0.21444711089134216, "p95": 0.4301546812057495 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.544264793395996, "group": 5, "latest_source_mass": 0.2861037850379944, "layer": 17, "max_source_mass": 0.2861037850379944, "mixer_index": 32, "normalized_entropy": 0.9595056643474137, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.24016115069389343, "median": 0.23868846893310547, "p05": 0.18435582518577576, "p95": 0.2983453571796417 }, { "label": "completed_group_1", "mean": 0.20730282366275787, "median": 0.1998077780008316, "p05": 0.16255050897598267, "p95": 0.28107208013534546 }, { "label": "completed_group_2", "mean": 0.09692573547363281, "median": 0.0967404842376709, "p05": 0.08098706603050232, "p95": 0.11221462488174438 }, { "label": "completed_group_3", "mean": 0.1695065200328827, "median": 0.1696585714817047, "p05": 0.136396586894989, "p95": 0.20130634307861328 }, { "label": "completed_group_4", "mean": 0.2861037850379944, "median": 0.28883200883865356, "p05": 0.22401469945907593, "p95": 0.3409370183944702 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.686682939529419, "group": 5, "latest_source_mass": 0.12281867861747742, "layer": 17, "max_source_mass": 0.2885604202747345, "mixer_index": 33, "normalized_entropy": 0.9413556721740635, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.16755425930023193, "median": 0.1327776312828064, "p05": 0.0790192037820816, "p95": 0.3667412996292114 }, { "label": "completed_group_1", "mean": 0.10962115228176117, "median": 0.10525335371494293, "p05": 0.07840172946453094, "p95": 0.15405339002609253 }, { "label": "completed_group_2", "mean": 0.1783454418182373, "median": 0.17896884679794312, "p05": 0.10802385956048965, "p95": 0.24048911035060883 }, { "label": "completed_group_3", "mean": 0.13310004770755768, "median": 0.13033035397529602, "p05": 0.0904831737279892, "p95": 0.17934636771678925 }, { "label": "completed_group_4", "mean": 0.2885604202747345, "median": 0.2831825315952301, "p05": 0.16967125236988068, "p95": 0.4189499616622925 }, { "label": "current_group_partial", "mean": 0.12281867861747742, "median": 0.12062528729438782, "p05": 0.07461707293987274, "p95": 0.1782219111919403 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.7070834636688232, "group": 5, "latest_source_mass": 0.2988377809524536, "layer": 18, "max_source_mass": 0.2988377809524536, "mixer_index": 34, "normalized_entropy": 0.9527414214834803, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.181177020072937, "median": 0.18419909477233887, "p05": 0.13624237477779388, "p95": 0.22192169725894928 }, { "label": "completed_group_1", "mean": 0.14557164907455444, "median": 0.144476979970932, "p05": 0.1293315887451172, "p95": 0.16486899554729462 }, { "label": "completed_group_2", "mean": 0.08318156749010086, "median": 0.0820261612534523, "p05": 0.07088497281074524, "p95": 0.0990293100476265 }, { "label": "completed_group_3", "mean": 0.11424209177494049, "median": 0.11267061531543732, "p05": 0.0979941189289093, "p95": 0.1353992223739624 }, { "label": "completed_group_4", "mean": 0.17698991298675537, "median": 0.17597860097885132, "p05": 0.14837387204170227, "p95": 0.21050144731998444 }, { "label": "current_group_partial", "mean": 0.2988377809524536, "median": 0.2989804744720459, "p05": 0.2721370458602905, "p95": 0.32572072744369507 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.7419235706329346, "group": 5, "latest_source_mass": 0.26016175746917725, "layer": 18, "max_source_mass": 0.26016175746917725, "mixer_index": 35, "normalized_entropy": 0.972186055410333, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1532188355922699, "median": 0.1451343297958374, "p05": 0.11514341086149216, "p95": 0.22103679180145264 }, { "label": "completed_group_1", "mean": 0.11420352011919022, "median": 0.11261284351348877, "p05": 0.0928458571434021, "p95": 0.14123988151550293 }, { "label": "completed_group_2", "mean": 0.16597424447536469, "median": 0.1655328869819641, "p05": 0.13871800899505615, "p95": 0.19611573219299316 }, { "label": "completed_group_3", "mean": 0.1250365972518921, "median": 0.12369821965694427, "p05": 0.10375828295946121, "p95": 0.14912742376327515 }, { "label": "completed_group_4", "mean": 0.18140508234500885, "median": 0.1796584278345108, "p05": 0.14511024951934814, "p95": 0.22286373376846313 }, { "label": "current_group_partial", "mean": 0.26016175746917725, "median": 0.26266711950302124, "p05": 0.1889389455318451, "p95": 0.3234381675720215 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.7127439975738525, "group": 5, "latest_source_mass": 0.2907913327217102, "layer": 19, "max_source_mass": 0.2907913327217102, "mixer_index": 36, "normalized_entropy": 0.9559006256078307, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.18731768429279327, "median": 0.19042012095451355, "p05": 0.13879702985286713, "p95": 0.23792000114917755 }, { "label": "completed_group_1", "mean": 0.16715484857559204, "median": 0.1667211651802063, "p05": 0.14574480056762695, "p95": 0.19026139378547668 }, { "label": "completed_group_2", "mean": 0.08496320247650146, "median": 0.08452782034873962, "p05": 0.07410907745361328, "p95": 0.097386933863163 }, { "label": "completed_group_3", "mean": 0.11362075805664062, "median": 0.11248823255300522, "p05": 0.0977015346288681, "p95": 0.13219426572322845 }, { "label": "completed_group_4", "mean": 0.15615218877792358, "median": 0.15540224313735962, "p05": 0.1289147436618805, "p95": 0.18598109483718872 }, { "label": "current_group_partial", "mean": 0.2907913327217102, "median": 0.2900415062904358, "p05": 0.25822532176971436, "p95": 0.32475030422210693 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.6076834201812744, "group": 5, "latest_source_mass": 0.3738824725151062, "layer": 19, "max_source_mass": 0.3738824725151062, "mixer_index": 37, "normalized_entropy": 0.8972652009334232, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.15032559633255005, "median": 0.12544110417366028, "p05": 0.06976518034934998, "p95": 0.35840943455696106 }, { "label": "completed_group_1", "mean": 0.08624927699565887, "median": 0.08602266013622284, "p05": 0.061998721212148666, "p95": 0.11138717085123062 }, { "label": "completed_group_2", "mean": 0.15736976265907288, "median": 0.15802210569381714, "p05": 0.10121746361255646, "p95": 0.20687000453472137 }, { "label": "completed_group_3", "mean": 0.08584291487932205, "median": 0.0827617198228836, "p05": 0.060561805963516235, "p95": 0.11351694166660309 }, { "label": "completed_group_4", "mean": 0.14632999897003174, "median": 0.1426895558834076, "p05": 0.10106660425662994, "p95": 0.20179018378257751 }, { "label": "current_group_partial", "mean": 0.3738824725151062, "median": 0.3799295425415039, "p05": 0.23631010949611664, "p95": 0.4884527325630188 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.709290862083435, "group": 5, "latest_source_mass": 0.2901708781719208, "layer": 20, "max_source_mass": 0.2901708781719208, "mixer_index": 38, "normalized_entropy": 0.9539733939957076, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.18991519510746002, "median": 0.1924753338098526, "p05": 0.14201581478118896, "p95": 0.23709386587142944 }, { "label": "completed_group_1", "mean": 0.17091524600982666, "median": 0.17006167769432068, "p05": 0.15297070145606995, "p95": 0.19153237342834473 }, { "label": "completed_group_2", "mean": 0.07822999358177185, "median": 0.07778562605381012, "p05": 0.06707640737295151, "p95": 0.09089364111423492 }, { "label": "completed_group_3", "mean": 0.116765096783638, "median": 0.11516137421131134, "p05": 0.10162825882434845, "p95": 0.13653048872947693 }, { "label": "completed_group_4", "mean": 0.1540035903453827, "median": 0.15287795662879944, "p05": 0.12593276798725128, "p95": 0.18590238690376282 }, { "label": "current_group_partial", "mean": 0.2901708781719208, "median": 0.29104921221733093, "p05": 0.25643545389175415, "p95": 0.3205650746822357 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.6690887212753296, "group": 5, "latest_source_mass": 0.3342471718788147, "layer": 20, "max_source_mass": 0.3342471718788147, "mixer_index": 39, "normalized_entropy": 0.9315361520005944, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.19029226899147034, "median": 0.19644704461097717, "p05": 0.09456793963909149, "p95": 0.30468082427978516 }, { "label": "completed_group_1", "mean": 0.1100241169333458, "median": 0.10808776319026947, "p05": 0.09035106748342514, "p95": 0.1359245330095291 }, { "label": "completed_group_2", "mean": 0.11471512168645859, "median": 0.11494918912649155, "p05": 0.08799737691879272, "p95": 0.14024731516838074 }, { "label": "completed_group_3", "mean": 0.09999909996986389, "median": 0.09682698547840118, "p05": 0.07921088486909866, "p95": 0.1264391839504242 }, { "label": "completed_group_4", "mean": 0.1507222205400467, "median": 0.1476232260465622, "p05": 0.11397356539964676, "p95": 0.19991227984428406 }, { "label": "current_group_partial", "mean": 0.3342471718788147, "median": 0.33802270889282227, "p05": 0.2548266053199768, "p95": 0.40130624175071716 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.7458693981170654, "group": 6, "latest_source_mass": 0.2361394464969635, "layer": 21, "max_source_mass": 0.2361394464969635, "mixer_index": 40, "normalized_entropy": 0.9743882636597644, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.20528709888458252, "median": 0.2140858769416809, "p05": 0.13168928027153015, "p95": 0.27098023891448975 }, { "label": "completed_group_1", "mean": 0.1722964197397232, "median": 0.17238952219486237, "p05": 0.15053534507751465, "p95": 0.19365043938159943 }, { "label": "completed_group_2", "mean": 0.10435444861650467, "median": 0.10361101478338242, "p05": 0.08888213336467743, "p95": 0.12212536484003067 }, { "label": "completed_group_3", "mean": 0.12075909972190857, "median": 0.1180175170302391, "p05": 0.10358668118715286, "p95": 0.14458929002285004 }, { "label": "completed_group_4", "mean": 0.16116347908973694, "median": 0.15841388702392578, "p05": 0.1337081342935562, "p95": 0.19791515171527863 }, { "label": "completed_group_5", "mean": 0.2361394464969635, "median": 0.23709020018577576, "p05": 0.20446214079856873, "p95": 0.26509594917297363 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.4612125158309937, "group": 6, "latest_source_mass": 0.5543455481529236, "layer": 21, "max_source_mass": 0.5543455481529236, "mixer_index": 41, "normalized_entropy": 0.7509146897354808, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.08934466540813446, "median": 0.07911523431539536, "p05": 0.04488275945186615, "p95": 0.1737232506275177 }, { "label": "completed_group_1", "mean": 0.054379627108573914, "median": 0.05344422906637192, "p05": 0.04221750423312187, "p95": 0.06924191862344742 }, { "label": "completed_group_2", "mean": 0.07567165046930313, "median": 0.07438549399375916, "p05": 0.05811355262994766, "p95": 0.09632256627082825 }, { "label": "completed_group_3", "mean": 0.05758609622716904, "median": 0.05612001568078995, "p05": 0.044792283326387405, "p95": 0.07347827404737473 }, { "label": "completed_group_4", "mean": 0.07884950935840607, "median": 0.07757881283760071, "p05": 0.05758974328637123, "p95": 0.10452556610107422 }, { "label": "completed_group_5", "mean": 0.08982288837432861, "median": 0.08912864327430725, "p05": 0.0663854256272316, "p95": 0.11604885756969452 }, { "label": "current_group_partial", "mean": 0.5543455481529236, "median": 0.5596089363098145, "p05": 0.45455610752105713, "p95": 0.6367484331130981 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.7241814136505127, "group": 6, "latest_source_mass": 0.39484071731567383, "layer": 22, "max_source_mass": 0.39484071731567383, "mixer_index": 42, "normalized_entropy": 0.886053970419732, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.13360010087490082, "median": 0.14072942733764648, "p05": 0.07699814438819885, "p95": 0.17506597936153412 }, { "label": "completed_group_1", "mean": 0.10609637200832367, "median": 0.10586297512054443, "p05": 0.08648421615362167, "p95": 0.12602579593658447 }, { "label": "completed_group_2", "mean": 0.06363911926746368, "median": 0.06324908882379532, "p05": 0.054980404675006866, "p95": 0.07316673547029495 }, { "label": "completed_group_3", "mean": 0.07426095008850098, "median": 0.07362888753414154, "p05": 0.06318790465593338, "p95": 0.08665015548467636 }, { "label": "completed_group_4", "mean": 0.09383076429367065, "median": 0.09294568747282028, "p05": 0.07638302445411682, "p95": 0.11445100605487823 }, { "label": "completed_group_5", "mean": 0.13373197615146637, "median": 0.13327741622924805, "p05": 0.11399447172880173, "p95": 0.15641780197620392 }, { "label": "current_group_partial", "mean": 0.39484071731567383, "median": 0.3879151940345764, "p05": 0.3302172124385834, "p95": 0.4852263331413269 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.3979300260543823, "group": 6, "latest_source_mass": 0.5830056071281433, "layer": 22, "max_source_mass": 0.5830056071281433, "mixer_index": 43, "normalized_entropy": 0.7183939231382495, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.08959861099720001, "median": 0.07544538378715515, "p05": 0.04165133833885193, "p95": 0.20099139213562012 }, { "label": "completed_group_1", "mean": 0.05414731800556183, "median": 0.05336771905422211, "p05": 0.04069708660244942, "p95": 0.07046990841627121 }, { "label": "completed_group_2", "mean": 0.07668881118297577, "median": 0.07607124000787735, "p05": 0.0555989071726799, "p95": 0.09980960935354233 }, { "label": "completed_group_3", "mean": 0.05603286623954773, "median": 0.054573021829128265, "p05": 0.04162490367889404, "p95": 0.0730118378996849 }, { "label": "completed_group_4", "mean": 0.06979228556156158, "median": 0.06884932518005371, "p05": 0.04576495662331581, "p95": 0.09806846082210541 }, { "label": "completed_group_5", "mean": 0.07073447108268738, "median": 0.07023533433675766, "p05": 0.04956750199198723, "p95": 0.09379896521568298 }, { "label": "current_group_partial", "mean": 0.5830056071281433, "median": 0.5862616896629333, "p05": 0.4658474922180176, "p95": 0.6849814057350159 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.7459560632705688, "group": 6, "latest_source_mass": 0.37691548466682434, "layer": 23, "max_source_mass": 0.37691548466682434, "mixer_index": 44, "normalized_entropy": 0.897243926765161, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.14318783581256866, "median": 0.1493408977985382, "p05": 0.08292607218027115, "p95": 0.19464610517024994 }, { "label": "completed_group_1", "mean": 0.1094653457403183, "median": 0.10858799517154694, "p05": 0.09432064741849899, "p95": 0.1279931515455246 }, { "label": "completed_group_2", "mean": 0.06349228322505951, "median": 0.06325754523277283, "p05": 0.05439673364162445, "p95": 0.0737837627530098 }, { "label": "completed_group_3", "mean": 0.07859592139720917, "median": 0.0775163471698761, "p05": 0.06603655964136124, "p95": 0.09357087314128876 }, { "label": "completed_group_4", "mean": 0.09567482024431229, "median": 0.09468592703342438, "p05": 0.07671141624450684, "p95": 0.11774884164333344 }, { "label": "completed_group_5", "mean": 0.13266831636428833, "median": 0.13199329376220703, "p05": 0.11244359612464905, "p95": 0.15504513680934906 }, { "label": "current_group_partial", "mean": 0.37691548466682434, "median": 0.37180638313293457, "p05": 0.31207239627838135, "p95": 0.4595233201980591 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.4055393934249878, "group": 6, "latest_source_mass": 0.5772827863693237, "layer": 23, "max_source_mass": 0.5772827863693237, "mixer_index": 45, "normalized_entropy": 0.7223043644164862, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.10154955089092255, "median": 0.08766824007034302, "p05": 0.04901552200317383, "p95": 0.20287738740444183 }, { "label": "completed_group_1", "mean": 0.05619276314973831, "median": 0.055477917194366455, "p05": 0.041608311235904694, "p95": 0.07371112704277039 }, { "label": "completed_group_2", "mean": 0.07791627198457718, "median": 0.07736755907535553, "p05": 0.055920060724020004, "p95": 0.10158214718103409 }, { "label": "completed_group_3", "mean": 0.053915880620479584, "median": 0.05215863138437271, "p05": 0.03798118978738785, "p95": 0.07156119495630264 }, { "label": "completed_group_4", "mean": 0.06398355215787888, "median": 0.06229078769683838, "p05": 0.040616974234580994, "p95": 0.09270931780338287 }, { "label": "completed_group_5", "mean": 0.06915920227766037, "median": 0.0684593990445137, "p05": 0.046986356377601624, "p95": 0.09408729523420334 }, { "label": "current_group_partial", "mean": 0.5772827863693237, "median": 0.5824689269065857, "p05": 0.4518115520477295, "p95": 0.6926202178001404 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.7853620052337646, "group": 6, "latest_source_mass": 0.3407441973686218, "layer": 24, "max_source_mass": 0.3407441973686218, "mixer_index": 46, "normalized_entropy": 0.9174945750195659, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.15284113585948944, "median": 0.1593906283378601, "p05": 0.08678612112998962, "p95": 0.211342453956604 }, { "label": "completed_group_1", "mean": 0.12206703424453735, "median": 0.12153112143278122, "p05": 0.1060115247964859, "p95": 0.1399766057729721 }, { "label": "completed_group_2", "mean": 0.06508444249629974, "median": 0.06482555717229843, "p05": 0.055350616574287415, "p95": 0.07557765394449234 }, { "label": "completed_group_3", "mean": 0.08347313106060028, "median": 0.08190013468265533, "p05": 0.06966532766819, "p95": 0.10050469636917114 }, { "label": "completed_group_4", "mean": 0.09811970591545105, "median": 0.09664031863212585, "p05": 0.07688035070896149, "p95": 0.12369741499423981 }, { "label": "completed_group_5", "mean": 0.13767032325267792, "median": 0.1373325139284134, "p05": 0.11642646044492722, "p95": 0.1606694608926773 }, { "label": "current_group_partial", "mean": 0.3407441973686218, "median": 0.33805108070373535, "p05": 0.271301805973053, "p95": 0.41781002283096313 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.509842872619629, "group": 6, "latest_source_mass": 0.5227814316749573, "layer": 24, "max_source_mass": 0.5227814316749573, "mixer_index": 47, "normalized_entropy": 0.7759057494780099, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.11788781732320786, "median": 0.10097194463014603, "p05": 0.06265684217214584, "p95": 0.2408325970172882 }, { "label": "completed_group_1", "mean": 0.06822571158409119, "median": 0.0673636943101883, "p05": 0.05209728330373764, "p95": 0.08648915588855743 }, { "label": "completed_group_2", "mean": 0.08556114137172699, "median": 0.08463940769433975, "p05": 0.06234566867351532, "p95": 0.1125786080956459 }, { "label": "completed_group_3", "mean": 0.06067090481519699, "median": 0.058974385261535645, "p05": 0.043267179280519485, "p95": 0.07997901737689972 }, { "label": "completed_group_4", "mean": 0.07228855788707733, "median": 0.06972275674343109, "p05": 0.04689513146877289, "p95": 0.106512151658535 }, { "label": "completed_group_5", "mean": 0.07258446514606476, "median": 0.07082539796829224, "p05": 0.050949547439813614, "p95": 0.09970614314079285 }, { "label": "current_group_partial", "mean": 0.5227814316749573, "median": 0.5290273427963257, "p05": 0.38773614168167114, "p95": 0.6392589807510376 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.839367389678955, "group": 7, "latest_source_mass": 0.29583215713500977, "layer": 25, "max_source_mass": 0.29583215713500977, "mixer_index": 48, "normalized_entropy": 0.9452478525649903, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.15906387567520142, "median": 0.15846745669841766, "p05": 0.09519976377487183, "p95": 0.24517810344696045 }, { "label": "completed_group_1", "mean": 0.13139380514621735, "median": 0.1308426856994629, "p05": 0.11478274315595627, "p95": 0.15000829100608826 }, { "label": "completed_group_2", "mean": 0.07717745751142502, "median": 0.07715500891208649, "p05": 0.06616044044494629, "p95": 0.08791667222976685 }, { "label": "completed_group_3", "mean": 0.09607303142547607, "median": 0.09436552226543427, "p05": 0.08152313530445099, "p95": 0.11384639143943787 }, { "label": "completed_group_4", "mean": 0.1107850894331932, "median": 0.1089654266834259, "p05": 0.08814431726932526, "p95": 0.139452263712883 }, { "label": "completed_group_5", "mean": 0.12967455387115479, "median": 0.1299237608909607, "p05": 0.10832332074642181, "p95": 0.15138982236385345 }, { "label": "completed_group_6", "mean": 0.29583215713500977, "median": 0.29647016525268555, "p05": 0.2276897132396698, "p95": 0.3604946434497833 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.5841834545135498, "group": 7, "latest_source_mass": 0.40767741203308105, "layer": 25, "max_source_mass": 0.40767741203308105, "mixer_index": 49, "normalized_entropy": 0.7618312045616817, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.07896879315376282, "median": 0.06809917092323303, "p05": 0.025932887569069862, "p95": 0.19368338584899902 }, { "label": "completed_group_1", "mean": 0.048752427101135254, "median": 0.04767637699842453, "p05": 0.034443892538547516, "p95": 0.0667153000831604 }, { "label": "completed_group_2", "mean": 0.055707164108753204, "median": 0.0549345538020134, "p05": 0.038696639239788055, "p95": 0.0750277042388916 }, { "label": "completed_group_3", "mean": 0.04139826446771622, "median": 0.03878612071275711, "p05": 0.026673054322600365, "p95": 0.05746251344680786 }, { "label": "completed_group_4", "mean": 0.04449018836021423, "median": 0.04196460545063019, "p05": 0.026162121444940567, "p95": 0.07164917886257172 }, { "label": "completed_group_5", "mean": 0.04285799711942673, "median": 0.04109475016593933, "p05": 0.029393533244729042, "p95": 0.06277710199356079 }, { "label": "completed_group_6", "mean": 0.2801477909088135, "median": 0.2763082981109619, "p05": 0.1455708146095276, "p95": 0.4254714846611023 }, { "label": "current_group_partial", "mean": 0.40767741203308105, "median": 0.4120897948741913, "p05": 0.21399420499801636, "p95": 0.5853031277656555 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.8125314712524414, "group": 7, "latest_source_mass": 0.3700707256793976, "layer": 26, "max_source_mass": 0.3700707256793976, "mixer_index": 50, "normalized_entropy": 0.8716433883436914, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.10889513790607452, "median": 0.11333495378494263, "p05": 0.04850132763385773, "p95": 0.1620434820652008 }, { "label": "completed_group_1", "mean": 0.08656036853790283, "median": 0.08640051633119583, "p05": 0.0650845319032669, "p95": 0.1084962785243988 }, { "label": "completed_group_2", "mean": 0.048123255372047424, "median": 0.04800213500857353, "p05": 0.03576180338859558, "p95": 0.06107594817876816 }, { "label": "completed_group_3", "mean": 0.06561170518398285, "median": 0.06379580497741699, "p05": 0.050968848168849945, "p95": 0.08332888036966324 }, { "label": "completed_group_4", "mean": 0.06917259097099304, "median": 0.06724178791046143, "p05": 0.050207968801259995, "p95": 0.09373342245817184 }, { "label": "completed_group_5", "mean": 0.08286689221858978, "median": 0.08183546364307404, "p05": 0.06535022705793381, "p95": 0.10340933501720428 }, { "label": "completed_group_6", "mean": 0.16869932413101196, "median": 0.16509854793548584, "p05": 0.11059704422950745, "p95": 0.23654375970363617 }, { "label": "current_group_partial", "mean": 0.3700707256793976, "median": 0.3694133758544922, "p05": 0.23441238701343536, "p95": 0.5025027990341187 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.50089693069458, "group": 7, "latest_source_mass": 0.49096518754959106, "layer": 26, "max_source_mass": 0.49096518754959106, "mixer_index": 51, "normalized_entropy": 0.7217788529328457, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.07974998652935028, "median": 0.07402504235506058, "p05": 0.04579866677522659, "p95": 0.1359037309885025 }, { "label": "completed_group_1", "mean": 0.04166398197412491, "median": 0.04038450866937637, "p05": 0.027992289513349533, "p95": 0.05969971418380737 }, { "label": "completed_group_2", "mean": 0.04607272148132324, "median": 0.044438350945711136, "p05": 0.030038591474294662, "p95": 0.06808309257030487 }, { "label": "completed_group_3", "mean": 0.03586764633655548, "median": 0.03317485377192497, "p05": 0.02203454077243805, "p95": 0.05572832003235817 }, { "label": "completed_group_4", "mean": 0.04223340004682541, "median": 0.03930673003196716, "p05": 0.02465122565627098, "p95": 0.06968094408512115 }, { "label": "completed_group_5", "mean": 0.04095568507909775, "median": 0.03919431194663048, "p05": 0.026239857077598572, "p95": 0.06058088690042496 }, { "label": "completed_group_6", "mean": 0.22249138355255127, "median": 0.2125401496887207, "p05": 0.10647299885749817, "p95": 0.3711881935596466 }, { "label": "current_group_partial", "mean": 0.49096518754959106, "median": 0.4988659620285034, "p05": 0.2878011167049408, "p95": 0.6676807403564453 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.8059929609298706, "group": 7, "latest_source_mass": 0.37139976024627686, "layer": 27, "max_source_mass": 0.37139976024627686, "mixer_index": 52, "normalized_entropy": 0.8684990295379666, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.10711923241615295, "median": 0.10972198843955994, "p05": 0.04896187782287598, "p95": 0.16201043128967285 }, { "label": "completed_group_1", "mean": 0.07651707530021667, "median": 0.07666032761335373, "p05": 0.05141128972172737, "p95": 0.10152198374271393 }, { "label": "completed_group_2", "mean": 0.058284737169742584, "median": 0.05812416970729828, "p05": 0.039273619651794434, "p95": 0.07745087146759033 }, { "label": "completed_group_3", "mean": 0.06609587371349335, "median": 0.06383530795574188, "p05": 0.04481930285692215, "p95": 0.09336265176534653 }, { "label": "completed_group_4", "mean": 0.07139429450035095, "median": 0.0694514587521553, "p05": 0.0485839769244194, "p95": 0.10034126043319702 }, { "label": "completed_group_5", "mean": 0.07493151724338531, "median": 0.07319007813930511, "p05": 0.051876723766326904, "p95": 0.10402432829141617 }, { "label": "completed_group_6", "mean": 0.17425751686096191, "median": 0.16937679052352905, "p05": 0.10449051856994629, "p95": 0.2624811828136444 }, { "label": "current_group_partial", "mean": 0.37139976024627686, "median": 0.3732905089855194, "p05": 0.23830299079418182, "p95": 0.5014542937278748 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.5622732639312744, "group": 7, "latest_source_mass": 0.4768889844417572, "layer": 27, "max_source_mass": 0.4768889844417572, "mixer_index": 53, "normalized_entropy": 0.7512946301290215, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.08775827288627625, "median": 0.08504067361354828, "p05": 0.0428270623087883, "p95": 0.144285649061203 }, { "label": "completed_group_1", "mean": 0.047765426337718964, "median": 0.04646281152963638, "p05": 0.031814731657505035, "p95": 0.06807165592908859 }, { "label": "completed_group_2", "mean": 0.053928617388010025, "median": 0.052262187004089355, "p05": 0.035437777638435364, "p95": 0.0784272700548172 }, { "label": "completed_group_3", "mean": 0.04293380305171013, "median": 0.03996016085147858, "p05": 0.02556057646870613, "p95": 0.06623870134353638 }, { "label": "completed_group_4", "mean": 0.04174989461898804, "median": 0.03903091698884964, "p05": 0.025013400241732597, "p95": 0.06869364529848099 }, { "label": "completed_group_5", "mean": 0.04734005779027939, "median": 0.04609068110585213, "p05": 0.029884062707424164, "p95": 0.06911227852106094 }, { "label": "completed_group_6", "mean": 0.20163494348526, "median": 0.19222964346408844, "p05": 0.09963390231132507, "p95": 0.3307305574417114 }, { "label": "current_group_partial", "mean": 0.4768889844417572, "median": 0.48383551836013794, "p05": 0.276212215423584, "p95": 0.6487575173377991 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.8757455348968506, "group": 7, "latest_source_mass": 0.3109493851661682, "layer": 28, "max_source_mass": 0.3109493851661682, "mixer_index": 54, "normalized_entropy": 0.9020429270551009, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.11563669890165329, "median": 0.12087491154670715, "p05": 0.05733206868171692, "p95": 0.1688953936100006 }, { "label": "completed_group_1", "mean": 0.08108697831630707, "median": 0.08185897022485733, "p05": 0.05175650864839554, "p95": 0.10753051191568375 }, { "label": "completed_group_2", "mean": 0.07280858606100082, "median": 0.07357072830200195, "p05": 0.04870978742837906, "p95": 0.0948689877986908 }, { "label": "completed_group_3", "mean": 0.07553227245807648, "median": 0.07362619787454605, "p05": 0.056317005306482315, "p95": 0.09756095707416534 }, { "label": "completed_group_4", "mean": 0.07606014609336853, "median": 0.07456053793430328, "p05": 0.05528895929455757, "p95": 0.1022825762629509 }, { "label": "completed_group_5", "mean": 0.06935927271842957, "median": 0.06843917071819305, "p05": 0.04978285729885101, "p95": 0.09167985618114471 }, { "label": "completed_group_6", "mean": 0.19856664538383484, "median": 0.19508783519268036, "p05": 0.1281973123550415, "p95": 0.27796900272369385 }, { "label": "current_group_partial", "mean": 0.3109493851661682, "median": 0.3105033040046692, "p05": 0.18172834813594818, "p95": 0.43916669487953186 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.6924852132797241, "group": 7, "latest_source_mass": 0.3315581679344177, "layer": 28, "max_source_mass": 0.3315581679344177, "mixer_index": 55, "normalized_entropy": 0.8139133413255193, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.08638577908277512, "median": 0.058608733117580414, "p05": 0.026544082909822464, "p95": 0.2740764319896698 }, { "label": "completed_group_1", "mean": 0.059912845492362976, "median": 0.05901291221380234, "p05": 0.03629481792449951, "p95": 0.08707985281944275 }, { "label": "completed_group_2", "mean": 0.06609142571687698, "median": 0.0647939145565033, "p05": 0.038691695779561996, "p95": 0.09703366458415985 }, { "label": "completed_group_3", "mean": 0.056745611131191254, "median": 0.05418796092271805, "p05": 0.032174739986658096, "p95": 0.08370660990476608 }, { "label": "completed_group_4", "mean": 0.06277773529291153, "median": 0.0600711889564991, "p05": 0.03501531109213829, "p95": 0.10094858705997467 }, { "label": "completed_group_5", "mean": 0.05352145433425903, "median": 0.05179818347096443, "p05": 0.032271772623062134, "p95": 0.08084158599376678 }, { "label": "completed_group_6", "mean": 0.2830069661140442, "median": 0.2759752869606018, "p05": 0.12981083989143372, "p95": 0.46204638481140137 }, { "label": "current_group_partial", "mean": 0.3315581679344177, "median": 0.3204721510410309, "p05": 0.13446375727653503, "p95": 0.5667697787284851 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.8980522155761719, "group": 8, "latest_source_mass": 0.20123262703418732, "layer": 29, "max_source_mass": 0.26602888107299805, "mixer_index": 56, "normalized_entropy": 0.9127701729200177, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.11603894829750061, "median": 0.09991352260112762, "p05": 0.030035395175218582, "p95": 0.2941743731498718 }, { "label": "completed_group_1", "mean": 0.08804173022508621, "median": 0.08880250155925751, "p05": 0.05664174258708954, "p95": 0.11779069900512695 }, { "label": "completed_group_2", "mean": 0.07975024729967117, "median": 0.07984748482704163, "p05": 0.05428915470838547, "p95": 0.10499341040849686 }, { "label": "completed_group_3", "mean": 0.09027000516653061, "median": 0.08898856490850449, "p05": 0.06189651042222977, "p95": 0.11996553838253021 }, { "label": "completed_group_4", "mean": 0.08430293202400208, "median": 0.0819794088602066, "p05": 0.05464346706867218, "p95": 0.12159879505634308 }, { "label": "completed_group_5", "mean": 0.07433462142944336, "median": 0.07287181913852692, "p05": 0.052520088851451874, "p95": 0.1010315865278244 }, { "label": "completed_group_6", "mean": 0.26602888107299805, "median": 0.2639757990837097, "p05": 0.15474572777748108, "p95": 0.3846370279788971 }, { "label": "completed_group_7", "mean": 0.20123262703418732, "median": 0.18140269815921783, "p05": 0.08419936150312424, "p95": 0.3782624900341034 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.7512003183364868, "group": 8, "latest_source_mass": 0.33482813835144043, "layer": 29, "max_source_mass": 0.33482813835144043, "mixer_index": 57, "normalized_entropy": 0.7970056117156376, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.07744461297988892, "median": 0.06970229744911194, "p05": 0.014401955530047417, "p95": 0.17144526541233063 }, { "label": "completed_group_1", "mean": 0.046086348593235016, "median": 0.04394831880927086, "p05": 0.028066720813512802, "p95": 0.0718703418970108 }, { "label": "completed_group_2", "mean": 0.05025433003902435, "median": 0.04861236736178398, "p05": 0.029644077643752098, "p95": 0.07761462777853012 }, { "label": "completed_group_3", "mean": 0.03748488053679466, "median": 0.03434944897890091, "p05": 0.020774798467755318, "p95": 0.05938909202814102 }, { "label": "completed_group_4", "mean": 0.04005061462521553, "median": 0.037498839199543, "p05": 0.019541850313544273, "p95": 0.06922487914562225 }, { "label": "completed_group_5", "mean": 0.041623204946517944, "median": 0.039552345871925354, "p05": 0.022081367671489716, "p95": 0.06821337342262268 }, { "label": "completed_group_6", "mean": 0.198423832654953, "median": 0.18318486213684082, "p05": 0.07548617571592331, "p95": 0.3721975088119507 }, { "label": "completed_group_7", "mean": 0.17380405962467194, "median": 0.1514919549226761, "p05": 0.05777408182621002, "p95": 0.36326974630355835 }, { "label": "current_group_partial", "mean": 0.33482813835144043, "median": 0.33602219820022583, "p05": 0.0795319601893425, "p95": 0.5979493856430054 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.0160694122314453, "group": 8, "latest_source_mass": 0.21711516380310059, "layer": 30, "max_source_mass": 0.21711516380310059, "mixer_index": 58, "normalized_entropy": 0.9175527313077866, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.09385271370410919, "median": 0.09211117774248123, "p05": 0.032377734780311584, "p95": 0.16496515274047852 }, { "label": "completed_group_1", "mean": 0.06208505108952522, "median": 0.06140740588307381, "p05": 0.0407925546169281, "p95": 0.0863042101264 }, { "label": "completed_group_2", "mean": 0.06560768187046051, "median": 0.06516212224960327, "p05": 0.041897669434547424, "p95": 0.09085705131292343 }, { "label": "completed_group_3", "mean": 0.07029052078723907, "median": 0.06780634075403214, "p05": 0.04846132919192314, "p95": 0.09917552024126053 }, { "label": "completed_group_4", "mean": 0.06935097277164459, "median": 0.06708167493343353, "p05": 0.04668639227747917, "p95": 0.09815650433301926 }, { "label": "completed_group_5", "mean": 0.056832700967788696, "median": 0.05595892667770386, "p05": 0.038822539150714874, "p95": 0.0771557092666626 }, { "label": "completed_group_6", "mean": 0.19425815343856812, "median": 0.19145764410495758, "p05": 0.12249615788459778, "p95": 0.27496030926704407 }, { "label": "completed_group_7", "mean": 0.17060703039169312, "median": 0.16317546367645264, "p05": 0.09239593893289566, "p95": 0.2726203501224518 }, { "label": "current_group_partial", "mean": 0.21711516380310059, "median": 0.21198657155036926, "p05": 0.11019522696733475, "p95": 0.3391069769859314 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 1.7837426662445068, "group": 8, "latest_source_mass": 0.29026859998703003, "layer": 30, "max_source_mass": 0.29026859998703003, "mixer_index": 59, "normalized_entropy": 0.8118162725118464, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.08836327493190765, "median": 0.07904411852359772, "p05": 0.02631576545536518, "p95": 0.17534467577934265 }, { "label": "completed_group_1", "mean": 0.04953558370471001, "median": 0.04658038914203644, "p05": 0.025181813165545464, "p95": 0.08318150043487549 }, { "label": "completed_group_2", "mean": 0.04813014343380928, "median": 0.045740917325019836, "p05": 0.026714935898780823, "p95": 0.07843323051929474 }, { "label": "completed_group_3", "mean": 0.0373263955116272, "median": 0.03401809185743332, "p05": 0.01990434154868126, "p95": 0.06290259957313538 }, { "label": "completed_group_4", "mean": 0.037015400826931, "median": 0.03405017405748367, "p05": 0.018646355718374252, "p95": 0.06544196605682373 }, { "label": "completed_group_5", "mean": 0.03390160948038101, "median": 0.03198609501123428, "p05": 0.018189728260040283, "p95": 0.05570102855563164 }, { "label": "completed_group_6", "mean": 0.1840541511774063, "median": 0.17327937483787537, "p05": 0.07966600358486176, "p95": 0.32982927560806274 }, { "label": "completed_group_7", "mean": 0.23140482604503632, "median": 0.22406929731369019, "p05": 0.08831030130386353, "p95": 0.40350067615509033 }, { "label": "current_group_partial", "mean": 0.29026859998703003, "median": 0.2745037376880646, "p05": 0.10280418395996094, "p95": 0.5399085879325867 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 1.9973840713500977, "group": 8, "latest_source_mass": 0.19101998209953308, "layer": 31, "max_source_mass": 0.20831164717674255, "mixer_index": 60, "normalized_entropy": 0.9090486661912383, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.10261696577072144, "median": 0.10389091819524765, "p05": 0.01371697522699833, "p95": 0.21163305640220642 }, { "label": "completed_group_1", "mean": 0.06039813905954361, "median": 0.06085003539919853, "p05": 0.026107093319296837, "p95": 0.09348798543214798 }, { "label": "completed_group_2", "mean": 0.06671924144029617, "median": 0.06656201928853989, "p05": 0.034684643149375916, "p95": 0.09819839894771576 }, { "label": "completed_group_3", "mean": 0.08377155661582947, "median": 0.0804532840847969, "p05": 0.05448959022760391, "p95": 0.12039553374052048 }, { "label": "completed_group_4", "mean": 0.07002593576908112, "median": 0.06737451255321503, "p05": 0.044841229915618896, "p95": 0.10655929893255234 }, { "label": "completed_group_5", "mean": 0.05768897384405136, "median": 0.056077782064676285, "p05": 0.039495617151260376, "p95": 0.08242568373680115 }, { "label": "completed_group_6", "mean": 0.20831164717674255, "median": 0.2021285891532898, "p05": 0.11652229726314545, "p95": 0.31706833839416504 }, { "label": "completed_group_7", "mean": 0.1594475358724594, "median": 0.1546279788017273, "p05": 0.06593266129493713, "p95": 0.2735573351383209 }, { "label": "current_group_partial", "mean": 0.19101998209953308, "median": 0.1713225245475769, "p05": 0.06408411264419556, "p95": 0.3850628137588501 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 1.969987154006958, "group": 8, "latest_source_mass": 0.16864418983459473, "layer": 31, "max_source_mass": 0.24503237009048462, "mixer_index": 61, "normalized_entropy": 0.8965797917640489, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.08120635151863098, "median": 0.061164144426584244, "p05": 0.0259944386780262, "p95": 0.23622018098831177 }, { "label": "completed_group_1", "mean": 0.07894632965326309, "median": 0.07658770680427551, "p05": 0.0496862456202507, "p95": 0.11596276611089706 }, { "label": "completed_group_2", "mean": 0.08615730702877045, "median": 0.08467075228691101, "p05": 0.050409119576215744, "p95": 0.12811316549777985 }, { "label": "completed_group_3", "mean": 0.07609779387712479, "median": 0.0722871869802475, "p05": 0.04099677503108978, "p95": 0.11950768530368805 }, { "label": "completed_group_4", "mean": 0.058728985488414764, "median": 0.056080836802721024, "p05": 0.03323862701654434, "p95": 0.09344407916069031 }, { "label": "completed_group_5", "mean": 0.05076971277594566, "median": 0.04946017265319824, "p05": 0.03047419898211956, "p95": 0.07563400268554688 }, { "label": "completed_group_6", "mean": 0.24503237009048462, "median": 0.24110376834869385, "p05": 0.10613486170768738, "p95": 0.3991503119468689 }, { "label": "completed_group_7", "mean": 0.1544169783592224, "median": 0.1400495171546936, "p05": 0.06199025735259056, "p95": 0.2995939254760742 }, { "label": "current_group_partial", "mean": 0.16864418983459473, "median": 0.15222924947738647, "p05": 0.0543956533074379, "p95": 0.3329208791255951 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.071648359298706, "group": 8, "latest_source_mass": 0.11520501971244812, "layer": 32, "max_source_mass": 0.20955611765384674, "mixer_index": 62, "normalized_entropy": 0.9428478002054053, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.10569609701633453, "median": 0.08689188957214355, "p05": 0.03173588588833809, "p95": 0.24571751058101654 }, { "label": "completed_group_1", "mean": 0.09922641515731812, "median": 0.09795533865690231, "p05": 0.06407676637172699, "p95": 0.14120742678642273 }, { "label": "completed_group_2", "mean": 0.08180510997772217, "median": 0.080217644572258, "p05": 0.054607540369033813, "p95": 0.11442367732524872 }, { "label": "completed_group_3", "mean": 0.10646799206733704, "median": 0.10510502755641937, "p05": 0.07415887713432312, "p95": 0.14216329157352448 }, { "label": "completed_group_4", "mean": 0.0826798751950264, "median": 0.08022409677505493, "p05": 0.05491747707128525, "p95": 0.11860857903957367 }, { "label": "completed_group_5", "mean": 0.06615597754716873, "median": 0.06485182791948318, "p05": 0.04477996751666069, "p95": 0.09238749742507935 }, { "label": "completed_group_6", "mean": 0.20955611765384674, "median": 0.206287682056427, "p05": 0.129981130361557, "p95": 0.2996031939983368 }, { "label": "completed_group_7", "mean": 0.13320739567279816, "median": 0.12465103715658188, "p05": 0.06303662061691284, "p95": 0.23065781593322754 }, { "label": "current_group_partial", "mean": 0.11520501971244812, "median": 0.10386130958795547, "p05": 0.03912060335278511, "p95": 0.22735844552516937 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 1.9781701564788818, "group": 8, "latest_source_mass": 0.16112206876277924, "layer": 32, "max_source_mass": 0.22251944243907928, "mixer_index": 63, "normalized_entropy": 0.9003040366848136, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.09446848928928375, "median": 0.074150450527668, "p05": 0.042592622339725494, "p95": 0.19443637132644653 }, { "label": "completed_group_1", "mean": 0.07056106626987457, "median": 0.06868506222963333, "p05": 0.04818308353424072, "p95": 0.0982733890414238 }, { "label": "completed_group_2", "mean": 0.08033448457717896, "median": 0.08047192543745041, "p05": 0.048296794295310974, "p95": 0.11253741383552551 }, { "label": "completed_group_3", "mean": 0.0658978745341301, "median": 0.06256154924631119, "p05": 0.04223886504769325, "p95": 0.0967884361743927 }, { "label": "completed_group_4", "mean": 0.058234501630067825, "median": 0.0563921295106411, "p05": 0.03529864549636841, "p95": 0.08846980333328247 }, { "label": "completed_group_5", "mean": 0.04316519945859909, "median": 0.042268507182598114, "p05": 0.029443470761179924, "p95": 0.05996086448431015 }, { "label": "completed_group_6", "mean": 0.2036968618631363, "median": 0.19645483791828156, "p05": 0.09398645162582397, "p95": 0.3395216464996338 }, { "label": "completed_group_7", "mean": 0.22251944243907928, "median": 0.2130982130765915, "p05": 0.11867167800664902, "p95": 0.35465875267982483 }, { "label": "current_group_partial", "mean": 0.16112206876277924, "median": 0.14023703336715698, "p05": 0.06008770316839218, "p95": 0.3247341811656952 } ], "sources": 9 }, { "branch": "output", "entropy_mean": 1.6036605834960938, "group": null, "latest_source_mass": 0.31639498472213745, "layer": null, "max_source_mass": 0.3412485122680664, "mixer_index": null, "normalized_entropy": 0.7298573846467136, "offset": null, "source_summaries": [ { "label": "embedding", "mean": 0.09759747982025146, "median": 0.09915056824684143, "p05": 0.025461740791797638, "p95": 0.16398483514785767 }, { "label": "completed_group_1", "mean": 0.04215940088033676, "median": 0.04050712287425995, "p05": 0.025720715522766113, "p95": 0.06398092210292816 }, { "label": "completed_group_2", "mean": 0.041145481169223785, "median": 0.03881259262561798, "p05": 0.023510465398430824, "p95": 0.0662468671798706 }, { "label": "completed_group_3", "mean": 0.029554028064012527, "median": 0.027420252561569214, "p05": 0.01627083122730255, "p95": 0.04924973472952843 }, { "label": "completed_group_4", "mean": 0.0289996936917305, "median": 0.027226522564888, "p05": 0.01560397818684578, "p95": 0.048351090401411057 }, { "label": "completed_group_5", "mean": 0.03647579252719879, "median": 0.0347352996468544, "p05": 0.020744025707244873, "p95": 0.058257561177015305 }, { "label": "completed_group_6", "mean": 0.06642461568117142, "median": 0.057809896767139435, "p05": 0.022837381809949875, "p95": 0.14078694581985474 }, { "label": "completed_group_7", "mean": 0.3412485122680664, "median": 0.3336990475654602, "p05": 0.0819513276219368, "p95": 0.6423859596252441 }, { "label": "completed_group_8", "mean": 0.31639498472213745, "median": 0.30713334679603577, "p05": 0.05841828137636185, "p95": 0.6156357526779175 } ], "sources": 9 } ], "mode": "uniform_groups_6_7_only", "optimizer_state_after": "ed03e6fbd4a12d8b063dcb22e0437754285f54d585374cd52fbd534f05d24637", "optimizer_state_before": "ed03e6fbd4a12d8b063dcb22e0437754285f54d585374cd52fbd534f05d24637", "optimizer_state_unchanged": true, "positions": { "attention_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "5e6ecbcd78543fc42d4098a51169dca446d404ce384015643e84d2c78433c949", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.0933690071105957, 0.24537226557731628, 0.07698935270309448, 0.0523640513420105, 0.09380833059549332, 0.03647654876112938, 0.039120711386203766, 0.06371714919805527, 0.05156813934445381, 0.03728010877966881, 0.03983496502041817, 0.040284499526023865, 0.046871740370988846, 0.04314620420336723, 0.04033593460917473, 0.04471700266003609, 0.08397205173969269, 0.0528092160820961, 0.04443390294909477, 0.07560808956623077, 0.03708760067820549, 0.03599191829562187, 0.033304885029792786, 0.04061548039317131, 0.025487789884209633, 0.021565141156315804, 0.02342291921377182, 0.017244853079319, 0.024166326969861984, 0.016819356009364128, 0.021590016782283783, 0.026201345026493073 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.6212446098506916e-05, "first_to_last_ratio": 1.0388659164913323, "imbalance_abs_log_ratio": 0.03812965325165574, "last_quartile_mean": 1.560590817462071e-05, "mean": 1.4617104483249932e-05, "normalized": [ 1.0292895181673478, 1.2016692961534428, 1.076082521206664, 0.9922137964283568, 1.2630225724956121, 1.178085197525959, 1.1292855587561677, 1.0034884137193194, 0.9804534177374618, 0.8924291836910767, 0.8463293877424031, 0.8165968950154805, 0.940446452300403, 0.8797824631093097, 0.839619571187272, 0.794435050016747, 0.860644322605403, 0.8135696439086648, 0.80946926267957, 0.7777586315606083, 1.1106198020806568, 1.090196045841461, 1.0743128240455424, 1.0590239397534222, 1.2538832645069269, 1.228361471308328, 1.1904219317756568, 1.1313101142103754, 1.4322947717244097, 1.0220688028220486, 0.6759488411564908, 0.606887034767412 ], "peak_layer": 29, "peak_normalized": 1.4322947717244097, "population_cv": 0.18594785545180623, "rest_mean": 1.429875700759348e-05, "spike_contrast": 1.1424895774590274, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.633618085179478e-05, "top_five_layers": [ 29, 5, 25, 26, 2 ], "top_five_spike_overlap": 1 }, "values": [ 1.5045232430566102e-05, 1.756492565618828e-05, 1.572921064507682e-05, 1.450329273211537e-05, 1.8461732906871475e-05, 1.7220194422407076e-05, 1.6506885003764182e-05, 1.4668094991066027e-05, 1.433139004802797e-05, 1.3044730621913914e-05, 1.237088508787565e-05, 1.1936282135138754e-05, 1.3746604054176714e-05, 1.2859872185799759e-05, 1.227280699822586e-05, 1.1612340131250676e-05, 1.2580127986439038e-05, 1.1892032489413396e-05, 1.183209678856656e-05, 1.1368579180270899e-05, 1.623404568817932e-05, 1.593550950929057e-05, 1.5703342796768993e-05, 1.5479863577638753e-05, 1.832814268709626e-05, 1.7955087969312444e-05, 1.7400521755916998e-05, 1.653647814237047e-05, 2.0936002329108305e-05, 1.4939686479920056e-05, 9.880414836516138e-06, 8.870931196724996e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 6.408976560123847e-05, "first_to_last_ratio": 1.0288313701266634, "imbalance_abs_log_ratio": 0.028423565994612907, "last_quartile_mean": 6.229375139810145e-05, "mean": 5.8220478877046844e-05, "normalized": [ 1.028497513598314, 1.1821859197303168, 1.065914149045944, 0.9900200492568093, 1.257328122595378, 1.1706634571026144, 1.114864869561888, 0.997016945963626, 0.9775141157001622, 0.8918094263101609, 0.8497881798482392, 0.8205551172883515, 0.948359848288721, 0.8877030805724055, 0.8457084532486584, 0.7995370581948489, 0.8727894897082312, 0.8192749616926154, 0.8134089409545415, 0.7785438686248053, 1.1094418137780861, 1.0906901930900577, 1.0728662431187908, 1.055815122961946, 1.249242653104195, 1.224199292926018, 1.187858420583586, 1.1303123428593111, 1.4361281238794703, 1.029044018252012, 0.6855858389996572, 0.6173323691602381 ], "peak_layer": 29, "peak_normalized": 1.4361281238794703, "population_cv": 0.18162200451320984, "rest_mean": 5.697400855650711e-05, "spike_contrast": 1.1400184093338321, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.495141860796139e-05, "top_five_layers": [ 29, 5, 25, 26, 27 ], "top_five_spike_overlap": 1 }, "values": [ 5.9879617765545845e-05, 6.882743036840111e-05, 6.205803219927475e-05, 5.7639441365608945e-05, 7.320224540308118e-05, 6.81565870763734e-05, 6.490796658908948e-05, 5.804680404253304e-05, 5.691133992513642e-05, 5.192157186684199e-05, 4.9475074774818495e-05, 4.7773111873539165e-05, 5.521396451513283e-05, 5.168249845155515e-05, 4.923755113850348e-05, 4.6549430408049375e-05, 5.081422204966657e-05, 4.769858060171828e-05, 4.735705806524493e-05, 4.532719685812481e-05, 6.459223368437961e-05, 6.350050534820184e-05, 6.246278644539416e-05, 6.147006206447259e-05, 7.273150549735874e-05, 7.127346907509491e-05, 6.91576860845089e-05, 6.580732588190585e-05, 8.361206710105762e-05, 5.991143552819267e-05, 3.991513585788198e-05, 3.594138615881093e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.05683544557541609, "first_to_last_ratio": 1.0288313840515713, "imbalance_abs_log_ratio": 0.028423579529297215, "last_quartile_mean": 0.05524272145703435, "mean": 0.0516305030323565, "normalized": [ 1.028497534729466, 1.1821859443074707, 1.0659141504464176, 0.9900200549875414, 1.2573281344406144, 1.1706634876341346, 1.1148648394529075, 0.9970169377389326, 0.977514082193386, 0.8918094296637383, 0.8497881604511922, 0.8205551261606459, 0.9483598388649065, 0.8877030640656113, 0.8457084914244615, 0.7995370598061192, 0.8727894934488823, 0.8192749894857447, 0.8134088869152266, 0.7785438862005972, 1.1094418278146725, 1.0906902282187974, 1.0728662262496005, 1.0558151260987747, 1.2492426809523147, 1.2241992767014935, 1.187858388506022, 1.1303124136504583, 1.4361280606029487, 1.0290439486083425, 0.6855858473754668, 0.6173323828031113 ], "peak_layer": 29, "peak_normalized": 1.4361280606029487, "population_cv": 0.1816220012729493, "rest_mean": 0.05052512056297726, "spike_contrast": 1.140018424997307, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.057599568367004396, "top_five_layers": [ 29, 5, 25, 26, 27 ], "top_five_spike_overlap": 1 }, "values": [ 0.05310184508562088, 0.0610368549823761, 0.05503368377685547, 0.051115233451128006, 0.06491648405790329, 0.06044194474816322, 0.057561032474040985, 0.05147648602724075, 0.050469543784856796, 0.046044569462537766, 0.043874990195035934, 0.042365673929452896, 0.04896429553627968, 0.0458325557410717, 0.0436643548309803, 0.04128050059080124, 0.045062560588121414, 0.042299579828977585, 0.04199671000242233, 0.04019661247730255, 0.05728103965520859, 0.056312885135412216, 0.05539262294769287, 0.05451226606965065, 0.06449902802705765, 0.0632060244679451, 0.06132972612977028, 0.05835859850049019, 0.0741480141878128, 0.05313005670905113, 0.03539714217185974, 0.0318731814622879 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 2.593757244540029e-06, "first_to_last_ratio": 0.6175460195507982, "imbalance_abs_log_ratio": 0.4820016876685613, "last_quartile_mean": 4.2001035751582094e-06, "mean": 3.1735052914427797e-06, "normalized": [ 0.9186304608157616, 0.81803237536734, 0.773816992313595, 0.7489400538113816, 0.842302029779543, 0.8193833603604069, 0.8094249330080334, 0.8079992913347175, 0.7750004654847595, 0.7354097843472472, 0.7333903283112595, 0.7363422763755016, 0.7724061816698488, 0.7762992911081323, 0.7891106496993121, 0.7969533984422099, 0.794371939527265, 0.8244484793721201, 0.8495647204770511, 0.8774029928641522, 1.2890895898164418, 1.3387973258399855, 1.3807955057968144, 1.4041649095493927, 1.562388064797019, 1.5474295027025924, 1.5131517692303262, 1.4440668273654098, 1.7160449824768609, 1.2384229238423676, 0.8351450191236544, 0.7312735749894963 ], "peak_layer": 29, "peak_normalized": 1.7160449824768609, "population_cv": 0.311060619380286, "rest_mean": 2.9413415884644364e-06, "spike_contrast": 1.5051598579670928, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.427189287525834e-06, "top_five_layers": [ 29, 25, 26, 27, 28 ], "top_five_spike_overlap": 1 }, "values": [ 2.9152786282793386e-06, 2.5960300717997598e-06, 2.4557123197155306e-06, 2.3767652237438597e-06, 2.673049948498374e-06, 2.6003174298239173e-06, 2.5687143079267116e-06, 2.564190026532742e-06, 2.459468078086502e-06, 2.3338268420047825e-06, 2.3274180875887396e-06, 2.336786110390676e-06, 2.451235104672378e-06, 2.463589908074937e-06, 2.5042468223546166e-06, 2.529135826989659e-06, 2.5209435534634395e-06, 2.6163916118093766e-06, 2.696098135857028e-06, 2.7844430405821186e-06, 4.090932634426281e-06, 4.248680397722637e-06, 4.3819618440466e-06, 4.45612477051327e-06, 4.958246790920384e-06, 4.910775714961346e-06, 4.801995146408444e-06, 4.582753717841115e-06, 5.44587783224415e-06, 3.930141701857792e-06, 2.6503371373109985e-06, 2.3207005597214447e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 4.810961490875343e-05, "first_to_last_ratio": 1.234093489968503, "imbalance_abs_log_ratio": 0.2103366843383024, "last_quartile_mean": 3.898376849065244e-05, "mean": 4.0807527568631485e-05, "normalized": [ 1.1146539334671366, 1.2600860664846947, 1.1424471545343187, 1.0649383556226828, 1.3556452724387436, 1.261682380998242, 1.181892330434246, 1.0501722234981232, 1.0370150651443946, 0.9446498254629279, 0.893185397969627, 0.8564962850199015, 1.0104531797223097, 0.9353779001492362, 0.8812953242903676, 0.8236036120098279, 0.9175331614628355, 0.8488465276325197, 0.8410527934809594, 0.7984454777510579, 1.0851219366671148, 1.0502727843588, 1.0190990958483477, 0.9835677700418947, 1.1879767974033957, 1.1534876315803766, 1.1166962639256897, 1.0635757024706907, 1.1587757242867365, 0.8238052686293769, 0.5948673255859301, 0.5432814316274942 ], "peak_layer": 5, "peak_normalized": 1.3556452724387436, "population_cv": 0.17969302986123725, "rest_mean": 4.031475644431474e-05, "spike_contrast": 1.0782278121903905, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.346849163994193e-05, "top_five_layers": [ 5, 6, 2, 25, 7 ], "top_five_spike_overlap": 1 }, "values": [ 4.54862711194437e-05, 5.142099689692259e-05, 4.6620443754363805e-05, 4.345750130596571e-05, 5.532053182832897e-05, 5.148613854544237e-05, 4.8230103857349604e-05, 4.2854931962210685e-05, 4.2318020859966055e-05, 3.854882379528135e-05, 3.644868775154464e-05, 3.495149576338008e-05, 4.12340959883295e-05, 3.8170459447428584e-05, 3.5963483242085204e-05, 3.360922710271552e-05, 3.7442259781528264e-05, 3.4639328077901155e-05, 3.4321285056648776e-05, 3.2582585845375434e-05, 4.428114334587008e-05, 4.2859035602305084e-05, 4.158691444899887e-05, 4.013696889160201e-05, 4.8478395910933614e-05, 4.707097832579166e-05, 4.5569613575935364e-05, 4.340189479989931e-05, 4.7286772314691916e-05, 3.3617456210777164e-05, 2.4275064788525924e-05, 2.216997199866455e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 3.471832769719185e-05, "first_to_last_ratio": 1.4476406261399848, "imbalance_abs_log_ratio": 0.36993507679277754, "last_quartile_mean": 2.3982697825886135e-05, "mean": 2.7817054530032692e-05, "normalized": [ 1.1923417182272795, 1.3212701945916308, 1.213797211027679, 1.1300899898707768, 1.4533898902411153, 1.3503969856834142, 1.2309702369008337, 1.0925040678178972, 1.0913606380318261, 0.9849745862935676, 0.9220150640445547, 0.8783581615062225, 1.0512619299564514, 0.968759279318205, 0.9095830278974506, 0.8330069740681361, 0.9437572316809448, 0.8702758797877849, 0.8576532938067216, 0.8143892613447199, 1.0865764247504401, 1.0238836330342753, 0.9728207241319347, 0.9092990994900892, 1.1421392261798584, 1.106316642089417, 1.0754673171176905, 1.0359694883917498, 0.9134574539997445, 0.6462857924007069, 0.5038707166866915, 0.47375785963019007 ], "peak_layer": 5, "peak_normalized": 1.4533898902411153, "population_cv": 0.21272891497678073, "rest_mean": 2.7678258649959593e-05, "spike_contrast": 1.0320935519716714, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.8566552282427436e-05, "top_five_layers": [ 5, 6, 2, 7, 3 ], "top_five_spike_overlap": 0 }, "values": [ 3.316743459436111e-05, 3.67538450518623e-05, 3.376426320755854e-05, 3.143577487207949e-05, 4.042902583023533e-05, 3.756406658794731e-05, 3.4241966204717755e-05, 3.0390245228772983e-05, 3.035843838006258e-05, 2.7399091777624562e-05, 2.5647743314038962e-05, 2.4433336875517853e-05, 2.9243010430946015e-05, 2.6948029699269682e-05, 2.530192068661563e-05, 2.317180042155087e-05, 2.6252546376781538e-05, 2.4208511604228988e-05, 2.3857388441683725e-05, 2.265391049149912e-05, 3.0225355658330955e-05, 2.8481426852522418e-05, 2.7061007131123915e-05, 2.5294022634625435e-05, 3.1770949135534465e-05, 3.0774470360483974e-05, 2.991633300553076e-05, 2.8817619750043377e-05, 2.5409695808775723e-05, 1.7977767129195854e-05, 1.4016199202160351e-05, 1.3178548215364572e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00013556160774896853, "first_to_last_ratio": 1.0205535129750223, "imbalance_abs_log_ratio": 0.020345139889040964, "last_quartile_mean": 0.000132831454720872, "mean": 0.00012411066222739464, "normalized": [ 1.0298156417139817, 1.1464925122946883, 1.0437136959884716, 0.9831556100875732, 1.2376149678461976, 1.1584861999104452, 1.136689280159688, 1.0021440521522922, 0.9831248320893022, 0.8993975380880705, 0.8685760988709486, 0.8438289467678306, 0.9649863903996901, 0.9036290731646422, 0.8712683236606603, 0.822828971985637, 0.8904009826315634, 0.8521332610746543, 0.8366920272792361, 0.8020923396191257, 1.054560448826755, 1.0477994899151355, 1.0342540049389344, 1.0241852026474245, 1.3059133978760782, 1.2842455698437834, 1.238913271750406, 1.1770723588815726, 1.3936529703566747, 0.9527779602931395, 0.6330867750028389, 0.5764678038825584 ], "peak_layer": 29, "peak_normalized": 1.3936529703566747, "population_cv": 0.18337942520570907, "rest_mean": 0.00012196532878550459, "spike_contrast": 1.1125740746556172, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001356954628136009, "top_five_layers": [ 29, 25, 26, 27, 5 ], "top_five_spike_overlap": 1 }, "values": [ 0.00012781110126525164, 0.00014229194493964314, 0.00012953599798493087, 0.0001220200938405469, 0.00015360121324192733, 0.00014378048945218325, 0.0001410752593073994, 0.0001243767619598657, 0.00012201627396279946, 0.00011162482405779883, 0.00010779955482576042, 0.0001047281693900004, 0.00011976509995292872, 0.00011215000267839059, 0.00010813368862727657, 0.00010212184861302376, 0.00011050825560232624, 0.00010575882333796471, 0.00010384240158600733, 9.954821143765002e-05, 0.0001308821956627071, 0.00013004308857489377, 0.00012836194946430624, 0.00012711230374407023, 0.00016207777662202716, 0.00015938856813590974, 0.00015376234659925103, 0.0001460872299503535, 0.0001729671930661425, 0.00011824990360764787, 7.857281889300793e-05, 7.154580089263618e-05 ] } } }, "mlp_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "c0b8018c349b1bd09e5d17858eb312a2323da0e92de9651e2bb08ae4adf115c7", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.13583384454250336, 0.0415157787501812, 0.018140379339456558, 0.039518751204013824, 0.022877270355820656, 0.015532293356955051, 0.02314678393304348, 0.05432315170764923, 0.03743875026702881, 0.04006272554397583, 0.045583780854940414, 0.05995568633079529, 0.038651175796985626, 0.04559725895524025, 0.05632574483752251, 0.07952121645212173, 0.042337141931056976, 0.05044656619429588, 0.04418695345520973, 0.06310147047042847, 0.029010163620114326, 0.04586305841803551, 0.06033914536237717, 0.07656749337911606, 0.045884523540735245, 0.051604680716991425, 0.06502768397331238, 0.06792232394218445, 0.060306549072265625, 0.08025261014699936, 0.09663844108581543, 0.12230555713176727 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.6079743431873794e-05, "first_to_last_ratio": 1.1717093632239473, "imbalance_abs_log_ratio": 0.1584636768044897, "last_quartile_mean": 1.3723320762437652e-05, "mean": 1.396758858618341e-05, "normalized": [ 1.0755475805659243, 1.2563584734596962, 1.127149120864466, 1.036192804442457, 1.3164329930304488, 1.219633027075646, 1.1544329423793873, 1.0239993691692295, 0.9603454970532628, 0.9028549874225551, 0.862410061757302, 0.8355072260845138, 0.959908577723112, 0.892395360048571, 0.8433127801504585, 0.8057146671698836, 0.8951210594763072, 0.837115492930325, 0.8302016840546058, 0.8204885310771713, 1.152353180322008, 1.126882020552651, 1.1134251656426255, 1.0921229064982418, 1.2861663994327341, 1.2462714321374442, 1.1851739613481123, 1.1309272033345117, 1.0695096940194433, 0.7116786229448732, 0.6375513191482172, 0.5928158586838146 ], "peak_layer": 5, "peak_normalized": 1.3164329930304488, "population_cv": 0.1881264826582693, "rest_mean": 1.3568762369686738e-05, "spike_contrast": 1.1881150038621862, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.6121250155265443e-05, "top_five_layers": [ 5, 25, 2, 26, 6 ], "top_five_spike_overlap": 1 }, "values": [ 1.5022806110209785e-05, 1.7548298274050467e-05, 1.574355519551318e-05, 1.4473114788415842e-05, 1.8387394447927363e-05, 1.7035332348314114e-05, 1.612464438949246e-05, 1.4302801901067141e-05, 1.3413710803433787e-05, 1.2610707017302047e-05, 1.2045788935211021e-05, 1.1670021194731817e-05, 1.340760809398489e-05, 1.2464611245377455e-05, 1.1779045962612145e-05, 1.1253890988882631e-05, 1.250268269359367e-05, 1.1692484804370906e-05, 1.1595915566431358e-05, 1.146024624176789e-05, 1.6095595128717832e-05, 1.573982444824651e-05, 1.555186463519931e-05, 1.5254323443514295e-05, 1.796464312064927e-05, 1.740740663080942e-05, 1.655402229516767e-05, 1.579632589709945e-05, 1.4938471394998487e-05, 9.940434210875537e-06, 8.905054528440814e-06, 8.280208021460567e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 6.355238247124362e-05, "first_to_last_ratio": 1.1581511119502832, "imbalance_abs_log_ratio": 0.14682486454937524, "last_quartile_mean": 5.4873998578841565e-05, "mean": 5.5652053788435296e-05, "normalized": [ 1.0734966800439847, 1.2354686199320681, 1.116051224659082, 1.033630348932252, 1.310105423501222, 1.2117213721091664, 1.1390813627834717, 1.0161196675338566, 0.9572509099626487, 0.9033960369744735, 0.8665232629761096, 0.8405712133381292, 0.9692210854716112, 0.902309913215059, 0.850216502208297, 0.8114691760025924, 0.9072964077375923, 0.8415746439554637, 0.8323813237685695, 0.8191196971015575, 1.150254938585409, 1.1260372877187066, 1.1097251012348526, 1.0888234780505945, 1.2815353024479814, 1.2427993506349422, 1.1825293201833968, 1.1304219199810577, 1.0768515375496668, 0.7214470682116813, 0.6484823657344362, 0.6040874574600676 ], "peak_layer": 5, "peak_normalized": 1.310105423501222, "population_cv": 0.1823587744141145, "rest_mean": 5.409302105329482e-05, "spike_contrast": 1.1844565030869407, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.407083055819385e-05, "top_five_layers": [ 5, 25, 26, 2, 6 ], "top_five_spike_overlap": 1 }, "values": [ 5.9742294979514554e-05, 6.875636609038338e-05, 6.211054278537631e-05, 5.752365177613683e-05, 7.29100574972108e-05, 6.743478297721595e-05, 6.339221727102995e-05, 5.654914639308117e-05, 5.327297913026996e-05, 5.027584484196268e-05, 4.8223799240076914e-05, 4.677951437770389e-05, 5.393914398155175e-05, 5.021539982408285e-05, 4.7316294512711465e-05, 4.515992623055354e-05, 5.0492908485466614e-05, 4.683535735239275e-05, 4.6323730202857405e-05, 4.558569344226271e-05, 6.401404971256852e-05, 6.266628770390525e-05, 6.175848102429882e-05, 6.059526276658289e-05, 7.132007158361375e-05, 6.916433630976826e-05, 6.581018533324823e-05, 6.291030149441212e-05, 5.992899968987331e-05, 4.015001104562543e-05, 3.608937549870461e-05, 3.36187076754868e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.05635888874530792, "first_to_last_ratio": 1.1581510956295176, "imbalance_abs_log_ratio": 0.14682485045728866, "last_quartile_mean": 0.0486628117505461, "mean": 0.049352797970641404, "normalized": [ 1.0734966916623554, 1.2354686411633367, 1.1160512857577445, 1.0336303441131571, 1.3101054129568566, 1.2117212807738589, 1.1390814080194662, 1.0161196794790694, 0.9572508995501673, 0.903395993058233, 0.8665232202205352, 0.8405711564806576, 0.9692211223217743, 0.9023099456806121, 0.850216507190079, 0.8114691930279554, 0.907296418839704, 0.8415746256103555, 0.8323812662615953, 0.8191196827354471, 1.1502548337297605, 1.1260372906105633, 1.1097251428142476, 1.0888234862151722, 1.281535301729809, 1.2427993854794188, 1.1825294172358949, 1.130421938933558, 1.0768515273378045, 0.7214470519829091, 0.648482376485543, 0.6040874725423581 ], "peak_layer": 5, "peak_normalized": 1.3101054129568566, "population_cv": 0.18235878029154182, "rest_mean": 0.04797023224333922, "spike_contrast": 1.1844564898049375, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.056818652898073196, "top_five_layers": [ 5, 25, 26, 2, 6 ], "top_five_spike_overlap": 1 }, "values": [ 0.05298006534576416, 0.06097383424639702, 0.05508025363087654, 0.0510125495493412, 0.06465736776590347, 0.05980183556675911, 0.056216854602098465, 0.05014834925532341, 0.04724301025271416, 0.04458511993288994, 0.04276534542441368, 0.041484538465738297, 0.047833774238824844, 0.04453152045607567, 0.04196056351065636, 0.040048275142908096, 0.04477761685848236, 0.04153406247496605, 0.04108034446835518, 0.040425848215818405, 0.05676829442381859, 0.05557309091091156, 0.05476804077625275, 0.05373648554086685, 0.06324735283851624, 0.06133562698960304, 0.05836113542318344, 0.055789485573768616, 0.0531456358730793, 0.035605430603027344, 0.03200441971421242, 0.029813406988978386 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 2.5718546510233864e-06, "first_to_last_ratio": 0.693614599114329, "imbalance_abs_log_ratio": 0.3658388054179276, "last_quartile_mean": 3.7079015555718797e-06, "mean": 3.01394216251083e-06, "normalized": [ 0.9610163737979156, 0.8596655174937965, 0.8155764329622244, 0.7901083574890673, 0.8863388323653948, 0.8548973683363681, 0.8379446532224404, 0.8210058946238301, 0.7674651561567859, 0.7529877233351875, 0.755429283668885, 0.7619352075848945, 0.7946006202869502, 0.7929449249265603, 0.7998446488963921, 0.8109461138236337, 0.8320840483984604, 0.8468576611072821, 0.867802995770728, 0.9090483460055899, 1.346163689885758, 1.3949739240664194, 1.438818656419001, 1.4595456659185078, 1.6181032977807608, 1.5801470078421707, 1.531660715698121, 1.4503677105645045, 1.2921221011203758, 0.8738530319999085, 0.7809594467750489, 0.7147845916770368 ], "peak_layer": 25, "peak_normalized": 1.6181032977807608, "population_cv": 0.294515580930307, "rest_mean": 2.761931362463791e-06, "spike_contrast": 1.5839642296042735, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.37480048276484e-06, "top_five_layers": [ 25, 26, 27, 24, 28 ], "top_five_spike_overlap": 2 }, "values": [ 2.8964477678528056e-06, 2.5909821488312446e-06, 2.4581001980550354e-06, 2.3813408915884793e-06, 2.6713739771366818e-06, 2.5766112230485305e-06, 2.525516720197629e-06, 2.474464281476685e-06, 2.313095592398895e-06, 2.2694614472129615e-06, 2.2768201688450063e-06, 2.296428647241555e-06, 2.3948803118400974e-06, 2.389890141785145e-06, 2.4106855107675074e-06, 2.444144683977356e-06, 2.5078531962208217e-06, 2.5523800104565453e-06, 2.6155080377066042e-06, 2.7398191377869807e-06, 4.05725950258784e-06, 4.204370725346962e-06, 4.3365162127884105e-06, 4.398986220621737e-06, 4.8768697524792515e-06, 4.762471689900849e-06, 4.61633680970408e-06, 4.371324394014664e-06, 3.8943812796787824e-06, 2.6337424969824497e-06, 2.353766603846452e-06, 2.1543194179685088e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 4.7774547965673264e-05, "first_to_last_ratio": 1.3606913118801394, "imbalance_abs_log_ratio": 0.30799288815728043, "last_quartile_mean": 3.511049681037548e-05, "mean": 3.9372213223032304e-05, "normalized": [ 1.1530478674954339, 1.304680409093685, 1.1851601654127024, 1.1021517366875562, 1.4004330598448003, 1.2951083595264692, 1.1990859941906606, 1.067594266774832, 1.0107748562644006, 0.9495623966910529, 0.9025340229234875, 0.8703861511401323, 1.0245046129804476, 0.943671179667803, 0.8784468195967535, 0.8287661138135604, 0.9456933460556104, 0.8665572019537329, 0.8558173131827017, 0.8399185670315442, 1.1165448305065537, 1.0757304257472786, 1.0442184483701784, 1.0055456827489706, 1.2071216223415429, 1.162690602796562, 1.106615194285428, 1.0581019581074413, 0.8612658434723904, 0.6260891861655198, 0.5689351503147435, 0.5432466148160243 ], "peak_layer": 5, "peak_normalized": 1.4004330598448003, "population_cv": 0.19807857457129563, "rest_mean": 3.871723311021924e-05, "spike_contrast": 1.108268912452248, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.2909105832222846e-05, "top_five_layers": [ 5, 2, 6, 25, 7 ], "top_five_spike_overlap": 1 }, "values": [ 4.539804649539292e-05, 5.136815525474958e-05, 4.666237873607315e-05, 4.339415318099782e-05, 5.513814903679304e-05, 5.099128247820772e-05, 4.7210669436026365e-05, 4.203354910714552e-05, 3.9796443161321804e-05, 3.738637315109372e-05, 3.5534761991584674e-05, 3.426902912906371e-05, 4.033701407024637e-05, 3.715442289831117e-05, 3.458639548625797e-05, 3.2630356145091355e-05, 3.723404006450437e-05, 3.411827492527664e-05, 3.3695421734591946e-05, 3.306945291114971e-05, 4.3960841139778495e-05, 4.235388769302517e-05, 4.1113191400654614e-05, 3.959055902669206e-05, 4.75270499009639e-05, 4.57777023257222e-05, 4.356988938525319e-05, 4.165981590631418e-05, 3.390994243090972e-05, 2.4650516934343614e-05, 2.2400236048270017e-05, 2.138882155122701e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 3.456835929682711e-05, "first_to_last_ratio": 1.5701420823225452, "imbalance_abs_log_ratio": 0.45116611356010783, "last_quartile_mean": 2.2016070829522505e-05, "mean": 2.7054395303593992e-05, "normalized": [ 1.2264874892922586, 1.3586185249135918, 1.252287796781162, 1.1618260300564736, 1.4890284911770828, 1.3766161313666212, 1.2418528641115851, 1.115166076876138, 1.0651386825368243, 0.9819688361654751, 0.9224896123736469, 0.8877556519124575, 1.0588958224690703, 0.97199714936962, 0.8993008955194284, 0.8356784786250799, 0.9676243503659734, 0.8799682803481665, 0.8694107119059236, 0.861738176909458, 1.110321963855831, 1.0464692687151116, 0.9879954700024759, 0.921198642268954, 1.1485539349225544, 1.1086782814356744, 1.0723020504793428, 1.0301741076566413, 0.6665838512414882, 0.5307028772513648, 0.4883733317069862, 0.4647961673875386 ], "peak_layer": 5, "peak_normalized": 1.4890284911770828, "population_cv": 0.23578280536843754, "rest_mean": 2.6839423804227004e-05, "spike_contrast": 1.0512610705052485, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.8215241400175728e-05, "top_five_layers": [ 5, 6, 2, 3, 7 ], "top_five_spike_overlap": 0 }, "values": [ 3.3181877370225266e-05, 3.6756602639798075e-05, 3.387988908798434e-05, 3.143250069115311e-05, 4.028476541861892e-05, 3.7243516999296844e-05, 3.3597578294575214e-05, 3.0170143872965127e-05, 2.881668297050055e-05, 2.6566573069430888e-05, 2.4957398636615835e-05, 2.4017692339839414e-05, 2.864778616640251e-05, 2.6296795113012195e-05, 2.4330041924258694e-05, 2.2608775907428935e-05, 2.617849168018438e-05, 2.380700971116312e-05, 2.3521381081081927e-05, 2.3313805286306888e-05, 3.0039089324418455e-05, 2.8311593268881552e-05, 2.6729620003607124e-05, 2.492247222107835e-05, 3.107343218289316e-05, 2.999462049046997e-05, 2.9010483558522537e-05, 2.7870737540069968e-05, 1.8034023014479317e-05, 1.435784542991314e-05, 1.3212645171734039e-05, 1.2574779248097911e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.000134735724714119, "first_to_last_ratio": 1.152305903847063, "imbalance_abs_log_ratio": 0.1417650685576954, "last_quartile_mean": 0.00011692704538290855, "mean": 0.00011863166059811192, "normalized": [ 1.0776052060931436, 1.1981690126194955, 1.0955956993922584, 1.0251810185270804, 1.2920863697147211, 1.2033252226357796, 1.1643845825280246, 1.029640493134955, 0.9576468724498742, 0.9195700983832678, 0.8862394662923972, 0.8617197158301618, 0.985882994860804, 0.9185275098966874, 0.8700799274076529, 0.8367668363666774, 0.9252824700794336, 0.8713572347681756, 0.8534192419540606, 0.8345916235150355, 1.0997681386382179, 1.0819903459021631, 1.0728406644147206, 1.0532810444704694, 1.342999696360823, 1.2963190373208089, 1.2306237575301286, 1.1708433688298545, 0.997331719198971, 0.6686640246699687, 0.6032570682203917, 0.5750095379937962 ], "peak_layer": 25, "peak_normalized": 1.342999696360823, "population_cv": 0.1880615651729425, "rest_mean": 0.00011577184718421281, "spike_contrast": 1.158093753309744, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013407465303316712, "top_five_layers": [ 25, 26, 5, 27, 6 ], "top_five_spike_overlap": 1 }, "values": [ 0.00012783809506800026, 0.00014214077964425087, 0.00012997233716305345, 0.00012161892664153129, 0.00015328235167544335, 0.00014275246940087527, 0.00013813287660013884, 0.00012214796151965857, 0.00011360723874531686, 0.00010909012780757621, 0.00010513605957385153, 0.0001022272408590652, 0.00011695693683577701, 0.0001089664438040927, 0.00010321902664145455, 9.926703933160752e-05, 0.00010976779594784603, 0.00010337055573472753, 0.00010124254185939208, 9.900899021886289e-05, 0.00013046732055954635, 0.00012835831148549914, 0.00012727286957670003, 0.00012495247938204557, 0.00015932228416204453, 0.00015378448006231338, 0.0001459909399272874, 0.00013889909314457327, 0.00011831511801574379, 7.932472362881526e-05, 7.156538777053356e-05, 6.821433635195717e-05 ] } } }, "post_attention_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "90a633f12d984935028fe5d7212be48e7da7b4a0806751939a551938d16dc540", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.0933690071105957, 0.35791492462158203, 0.3940892517566681, 0.4057151973247528, 0.09380833059549332, 0.11286329478025436, 0.13231655955314636, 0.1621972620487213, 0.05156813934445381, 0.08533531427383423, 0.12003090232610703, 0.16063299775123596, 0.046871740370988846, 0.079384446144104, 0.11459959298372269, 0.15625107288360596, 0.08397205173969269, 0.11654024571180344, 0.15294702351093292, 0.21572710573673248, 0.03708760067820549, 0.06553087383508682, 0.09390194714069366, 0.12852425873279572, 0.025487789884209633, 0.06059965491294861, 0.08860157430171967, 0.12100546807050705, 0.024166326969861984, 0.06877554953098297, 0.12682415544986725, 0.2025238573551178 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.6212396303671994e-05, "first_to_last_ratio": 1.038859236901509, "imbalance_abs_log_ratio": 0.03812322353710724, "last_quartile_mean": 1.5605960584252898e-05, "mean": 1.461712599848397e-05, "normalized": [ 1.0292623057921688, 1.2016835804550856, 1.07606575533391, 0.9922276423775954, 1.263018722350467, 1.178087694518937, 1.1292476838168954, 1.0035031763907987, 0.9804529701276157, 0.8924407498895575, 0.8463436973064455, 0.8165815688446151, 0.9404309438329486, 0.8797864569398384, 0.8396347617240242, 0.7944360584132357, 0.8606438024602251, 0.8135693174959712, 0.8094691289658219, 0.7777505179912517, 1.1106180428952754, 1.0901997921812876, 1.0743221936696181, 1.0590310919193364, 1.253901951878146, 1.2283408724652651, 1.190419432916244, 1.131324999844447, 1.4322710105340444, 1.0220922491086817, 0.6759643970457387, 0.6068774305145068 ], "peak_layer": 29, "peak_normalized": 1.4322710105340444, "population_cv": 0.1859455345692386, "rest_mean": 1.4298757917088181e-05, "spike_contrast": 1.1424987913459261, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.633631363802124e-05, "top_five_layers": [ 29, 5, 25, 26, 2 ], "top_five_spike_overlap": 1 }, "values": [ 1.504485680925427e-05, 1.7565160305821337e-05, 1.5728988728369586e-05, 1.4503516467812005e-05, 1.846170380304102e-05, 1.7220256268046796e-05, 1.650635567784775e-05, 1.466833236918319e-05, 1.4331404599943198e-05, 1.304491888731718e-05, 1.2371112461551093e-05, 1.1936075679841451e-05, 1.3746397598879412e-05, 1.285994949284941e-05, 1.2273047104827128e-05, 1.1612371963565238e-05, 1.258013890037546e-05, 1.189204522233922e-05, 1.183211224997649e-05, 1.13684773168643e-05, 1.6234043869189918e-05, 1.593558772583492e-05, 1.5703502867836505e-05, 1.5479990906897e-05, 1.8328442820347846e-05, 1.795481330191251e-05, 1.7400510841980577e-05, 1.653672006796114e-05, 2.0935685824952088e-05, 1.4940051187295467e-05, 9.88065676210681e-06, 8.870803867466748e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 6.408945682778722e-05, "first_to_last_ratio": 1.028824640923687, "imbalance_abs_log_ratio": 0.028417025345505106, "last_quartile_mean": 6.229385871847626e-05, "mean": 5.822048615300446e-05, "normalized": [ 1.0284745151035197, 1.1821852720997732, 1.0659053927360052, 0.9900198630453323, 1.2573302149683379, 1.1706710590946903, 1.1148396107691991, 0.9970215703175963, 0.9775055578964894, 0.8918176880136325, 0.8498026954264132, 0.8205452668893587, 0.9483574177792685, 0.8877083434499193, 0.845711284411295, 0.799530897110225, 0.8727927548901081, 0.8192735470949319, 0.8134159002450421, 0.778532836237653, 1.1094479237516128, 1.0906934310403467, 1.0728831052951668, 1.0558171155460176, 1.2492457462673283, 1.2241903918623305, 1.1878618963354504, 1.1303223243709826, 1.4361080737812122, 1.029058573914374, 0.6855976881906247, 0.6173320420657626 ], "peak_layer": 29, "peak_normalized": 1.4361080737812122, "population_cv": 0.1816200922729602, "rest_mean": 5.6973948193006904e-05, "spike_contrast": 1.1400261557608455, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.495179113699123e-05, "top_five_layers": [ 29, 5, 25, 26, 27 ], "top_five_spike_overlap": 1 }, "values": [ 5.987828626530245e-05, 6.882740126457065e-05, 6.205753015819937e-05, 5.763943772763014e-05, 7.320237637031823e-05, 6.815703818574548e-05, 6.490650412160903e-05, 5.804708052892238e-05, 5.691084879799746e-05, 5.192205935600214e-05, 4.9475926061859354e-05, 4.777254434884526e-05, 5.521382990991697e-05, 5.168281131773256e-05, 4.923772212350741e-05, 4.654907752410509e-05, 5.081441850052215e-05, 4.769850420416333e-05, 4.735746915685013e-05, 4.5326560211833566e-05, 6.459259748226032e-05, 6.350070179905742e-05, 6.246377597562969e-05, 6.147018575575203e-05, 7.273169467225671e-05, 7.127295975806192e-05, 6.915789708727971e-05, 6.58079152344726e-05, 8.361091022379696e-05, 5.9912290453212336e-05, 3.991583071183413e-05, 3.59413716068957e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.05683517036959529, "first_to_last_ratio": 1.0288246331365767, "imbalance_abs_log_ratio": 0.028417017776566716, "last_quartile_mean": 0.05524281645193696, "mean": 0.05163050885312259, "normalized": [ 1.0284745463124514, 1.1821853059589307, 1.0659053719299392, 0.990019871220895, 1.2573301572774662, 1.1706710760138022, 1.1148396045594258, 0.9970215152740425, 0.9775055301016954, 0.8918176988570897, 0.8498027116831899, 0.8205452930123577, 0.948357423055473, 0.8877083754535209, 0.8457112821957289, 0.7995309088246485, 0.8727927862372804, 0.8192735262167314, 0.8134159383482797, 0.7785328311895214, 1.1094479078857635, 1.0906934242884216, 1.0728830612244262, 1.0558170995010911, 1.249245714841137, 1.2241903360343511, 1.1878619343854537, 1.130322315471662, 1.4361081288042863, 1.0290586239370172, 0.6855976753097243, 0.6173320245941956 ], "peak_layer": 29, "peak_normalized": 1.4361081288042863, "population_cv": 0.18162009033028112, "rest_mean": 0.05052506675322851, "spike_contrast": 1.1400261275037322, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.05759989619255066, "top_five_layers": [ 29, 5, 25, 26, 27 ], "top_five_spike_overlap": 1 }, "values": [ 0.05310066416859627, 0.06103682890534401, 0.05503323674201965, 0.05111522972583771, 0.06491659581661224, 0.06044234335422516, 0.05755973607301712, 0.05147672817111015, 0.050469107925891876, 0.04604500159621239, 0.04387574642896652, 0.042365171015262604, 0.04896417632699013, 0.045832835137844086, 0.04366450384259224, 0.04128018766641617, 0.04506273567676544, 0.042299509048461914, 0.04199707880616188, 0.040196046233177185, 0.057281360030174255, 0.056313056498765945, 0.05539349839091301, 0.054512374103069305, 0.06449919193983078, 0.06320556998252869, 0.0613299161195755, 0.058359116315841675, 0.07414699345827103, 0.05313082039356232, 0.03539775684475899, 0.03187316656112671 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 2.5937198984138377e-06, "first_to_last_ratio": 0.617538582066725, "imbalance_abs_log_ratio": 0.4820137313521496, "last_quartile_mean": 4.200093684403328e-06, "mean": 3.1734995289411927e-06, "normalized": [ 0.9186288330923178, 0.8180092856340196, 0.7738031364832026, 0.7489205642949215, 0.8422756883251954, 0.8194431693718766, 0.8093776823985808, 0.8079888650111516, 0.7750328961632394, 0.7354366979173939, 0.7333900837679969, 0.7363406758877402, 0.7724261409524041, 0.7763269954023926, 0.7891217550097854, 0.796865572636081, 0.7943861352387754, 0.8244645208875896, 0.8495866110559664, 0.8774038695932518, 1.2890894945512135, 1.3387440150087604, 1.380842864479477, 1.4041774899253208, 1.5624033684965428, 1.547414830536607, 1.5131677000017953, 1.4440390709406175, 1.7159691428316337, 1.2384340568989498, 0.8351735467472807, 0.7313152404579184 ], "peak_layer": 29, "peak_normalized": 1.7159691428316337, "population_cv": 0.3110538203984811, "rest_mean": 2.9413336809132793e-06, "spike_contrast": 1.5051658834292099, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.427195108291926e-06, "top_five_layers": [ 29, 25, 26, 27, 28 ], "top_five_spike_overlap": 1 }, "values": [ 2.915268169090268e-06, 2.5959520826290827e-06, 2.455663889122661e-06, 2.3766990580043057e-06, 2.6729615001386264e-06, 2.600502511995728e-06, 2.5685596938274102e-06, 2.5641522825026186e-06, 2.459566530887969e-06, 2.3339080144069158e-06, 2.32741308536788e-06, 2.336776788069983e-06, 2.4512939944543177e-06, 2.4636733542138245e-06, 2.5042775178008014e-06, 2.528852519390057e-06, 2.5209840259776684e-06, 2.6164377686654916e-06, 2.696162709980854e-06, 2.784440766845364e-06, 4.090924903721316e-06, 4.2485035010031424e-06, 4.382104179967428e-06, 4.456156602827832e-06, 4.958286353939911e-06, 4.910720235784538e-06, 4.802036983164726e-06, 4.582657311402727e-06, 5.445627266453812e-06, 3.930169896193547e-06, 2.6504228571866406e-06, 2.3208285711007193e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 4.8109505769389216e-05, "first_to_last_ratio": 1.2340887973236232, "imbalance_abs_log_ratio": 0.2103328818275203, "last_quartile_mean": 3.898382828992908e-05, "mean": 4.080753672042192e-05, "normalized": [ 1.1146409350827386, 1.2600883692294655, 1.1424424408369784, 1.0649395431868474, 1.3556442552145924, 1.2616888734199336, 1.1818749486362035, 1.0501748407687133, 1.0370101967926515, 0.944652555548406, 0.8931932211285999, 0.856491992050668, 1.0104567865470482, 0.9353769771772559, 0.8812945025970124, 0.8236028924045083, 0.9175353627316157, 0.8488458915157222, 0.8410559925482458, 0.7984397714058472, 1.0851234763034818, 1.0502782543965719, 1.0191024332850536, 0.9835693324535263, 1.1879776899250407, 1.1534827371076617, 1.116697796480972, 1.0635802780282628, 1.1587762667585064, 0.8238077583672831, 0.5948716496606828, 0.5432819784099034 ], "peak_layer": 5, "peak_normalized": 1.3556442552145924, "population_cv": 0.17969218207139956, "rest_mean": 4.031474613670812e-05, "spike_contrast": 1.078230921387265, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.3468605872476475e-05, "top_five_layers": [ 5, 6, 2, 25, 7 ], "top_five_spike_overlap": 1 }, "values": [ 4.5485750888474286e-05, 5.1421102398307994e-05, 4.662026185542345e-05, 4.345755951362662e-05, 5.532050272449851e-05, 5.148641503183171e-05, 4.822940536541864e-05, 4.285504837753251e-05, 4.2317831685068086e-05, 3.8548943848581985e-05, 3.644901516963728e-05, 3.4951328416354954e-05, 4.1234252421418205e-05, 3.817043034359813e-05, 3.5963457776233554e-05, 3.360920527484268e-05, 3.7442358006956056e-05, 3.463930988800712e-05, 3.4321423299843445e-05, 3.2582360290689394e-05, 4.428121610544622e-05, 4.285926843294874e-05, 4.158705996815115e-05, 4.013704165117815e-05, 4.8478443204658106e-05, 4.707078915089369e-05, 4.5569686335511506e-05, 4.340209125075489e-05, 4.728680505650118e-05, 3.3617565350141376e-05, 2.427524668746628e-05, 2.2169999283505604e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 3.472003800197854e-05, "first_to_last_ratio": 1.447540825607753, "imbalance_abs_log_ratio": 0.36986613428680026, "last_quartile_mean": 2.39855328345584e-05, "mean": 2.7818581429528422e-05, "normalized": [ 1.1920650714197762, 1.3212213433361055, 1.213782636927801, 1.1301247353693578, 1.4536517015909163, 1.3505647994279242, 1.2307954360948288, 1.092498374403742, 1.0914587120618773, 0.9847036980715939, 0.9218339431200896, 0.8782324661293416, 1.0514178497355258, 0.9686061287032505, 0.9095250602556663, 0.8331112512675483, 0.9437670260544532, 0.8704111974903598, 0.857712931654502, 0.8145406586645088, 1.0864212537714835, 1.0238930180911465, 0.9728134918093693, 0.909146008520082, 1.1421506862241428, 1.106643274808871, 1.0757651724803503, 1.035786036018824, 0.9135118057050899, 0.6462377648109688, 0.5039411417069989, 0.4736653242735043 ], "peak_layer": 5, "peak_normalized": 1.4536517015909163, "population_cv": 0.21274903583876587, "rest_mean": 2.7680081513136867e-05, "spike_contrast": 1.0320230077532564, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.856648097804282e-05, "top_five_layers": [ 5, 6, 2, 7, 3 ], "top_five_spike_overlap": 0 }, "values": [ 3.316155925858766e-05, 3.675450352602638e-05, 3.3765711123123765e-05, 3.143846697639674e-05, 4.0438528230879456e-05, 3.757079684874043e-05, 3.423898306209594e-05, 3.0391754989977926e-05, 3.0362833058461547e-05, 2.7393060008762404e-05, 2.5644112611189485e-05, 2.443118137307465e-05, 2.92489530693274e-05, 2.694524846447166e-05, 2.5301696950918995e-05, 2.3175973183242604e-05, 2.625425986479968e-05, 2.4213604774558917e-05, 2.386035703239031e-05, 2.2659365640720353e-05, 3.0222698114812374e-05, 2.8483251298894174e-05, 2.706229133764282e-05, 2.529115226934664e-05, 3.1773011869518086e-05, 3.078524605371058e-05, 2.9926261049695313e-05, 2.8814098186558113e-05, 2.541260255384259e-05, 1.7977417883230373e-05, 1.401892768626567e-05, 1.3176697393646464e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.0001355614958811202, "first_to_last_ratio": 1.0205567795842085, "imbalance_abs_log_ratio": 0.020348340704987235, "last_quartile_mean": 0.00013283091993798735, "mean": 0.00012411249940669222, "normalized": [ 1.029565550527936, 1.1462858343914106, 1.0438524272154854, 0.9831832074722671, 1.2376896255013339, 1.158586650905878, 1.1367439754265505, 1.0020681317882287, 0.9831955771132294, 0.8993821728906659, 0.8685761390010557, 0.8437230094805183, 0.9650501931799182, 0.9034874280928744, 0.8713828163977875, 0.8228928858433737, 0.8902551952011143, 0.8520208694822519, 0.836552604146968, 0.8022113737694491, 1.0545713366601597, 1.0482675096806593, 1.034356646628807, 1.0241299433714284, 1.305699083989798, 1.2843858994923254, 1.2388245840275418, 1.17713489821688, 1.393508175362634, 0.9527629774081203, 0.6331337412743949, 0.5765195360589552 ], "peak_layer": 29, "peak_normalized": 1.393508175362634, "population_cv": 0.18336870951558631, "rest_mean": 0.00012196570012882283, "spike_contrast": 1.112650649845425, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013570521550718695, "top_five_layers": [ 29, 25, 26, 27, 5 ], "top_five_spike_overlap": 1 }, "values": [ 0.00012778195377904922, 0.00014226839994080365, 0.00012955513375345618, 0.0001220253252540715, 0.00015361275291070342, 0.0001437950850231573, 0.0001410841359756887, 0.0001243691804120317, 0.0001220268604811281, 0.00011162456939928234, 0.00010780115553643554, 0.0001047165715135634, 0.00011977479152847081, 0.00011213408288313076, 0.00010814949928317219, 0.00010213129280600697, 0.00011049179738620296, 0.00010574643965810537, 0.00010382663458585739, 9.95644586510025e-05, 0.0001308854843955487, 0.00013010310067329556, 0.00012837658869102597, 0.00012710732698906213, 0.00016205357678700238, 0.0001594083441887051, 0.000153753615450114, 0.00014609715435653925, 0.00017295178258791566, 0.00011824979446828365, 7.857981108827516e-05, 7.155328057706356e-05 ] } } }, "post_mlp_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "89f2226456bfe96e245d2aad0f699d32c676f9f53099789cd870253d002402f5", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.16915453970432281, 0.374446302652359, 0.39966297149658203, 0.4194087088108063, 0.09892481565475464, 0.11664740741252899, 0.13951337337493896, 0.18593810498714447, 0.06607726216316223, 0.10425819456577301, 0.14551182091236115, 0.19865906238555908, 0.06484340131282806, 0.10430748760700226, 0.14720691740512848, 0.20439592003822327, 0.10052042454481125, 0.14085541665554047, 0.1745898574590683, 0.2420409768819809, 0.049456026405096054, 0.08509506285190582, 0.11783231794834137, 0.15649692714214325, 0.05533855780959129, 0.08507220447063446, 0.11942370235919952, 0.1559232622385025, 0.06628894805908203, 0.12384890764951706, 0.19928614795207977, 0.30097496509552 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.60797850412564e-05, "first_to_last_ratio": 1.171713016473603, "imbalance_abs_log_ratio": 0.15846679468002706, "last_quartile_mean": 1.3723313486480038e-05, "mean": 1.3967615927867882e-05, "normalized": [ 1.0755471681533575, 1.2563643487905534, 1.1271458726314765, 1.0361757997482168, 1.3164404437428028, 1.2196456159784994, 1.1544236502017016, 1.0240092155287501, 0.9603339151072124, 0.9028478806877875, 0.8624088293865831, 0.8355184181363361, 0.9599245400771732, 0.8923995386017187, 0.8432978459976164, 0.8057107458560566, 0.895115595743916, 0.8371395745102641, 0.8301872313684935, 0.8204785251924319, 1.15237710062414, 1.1268946607870527, 1.1134237674811036, 1.0921147781252207, 1.2861703932076627, 1.246267169347558, 1.1851653903690358, 1.1309197803761892, 1.0694997215906317, 0.7116940944925246, 0.6375436899144004, 0.5928146982435332 ], "peak_layer": 5, "peak_normalized": 1.3164404437428028, "population_cv": 0.18812759928677, "rest_mean": 1.3568772879403292e-05, "spike_contrast": 1.188122797312651, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.6121368389576674e-05, "top_five_layers": [ 5, 25, 2, 26, 6 ], "top_five_spike_overlap": 1 }, "values": [ 1.5022829757072031e-05, 1.7548414689372294e-05, 1.5743540643597953e-05, 1.4472905604634434e-05, 1.8387534510111436e-05, 1.703554153209552e-05, 1.6124546164064668e-05, 1.4302967429102864e-05, 1.3413575288723223e-05, 1.2610632438736502e-05, 1.2045795301673934e-05, 1.1670200365188066e-05, 1.3407857295533177e-05, 1.2464694009395316e-05, 1.1778860425692983e-05, 1.1253858247073367e-05, 1.2502630852395669e-05, 1.1692844054778107e-05, 1.159573639597511e-05, 1.1460128916951362e-05, 1.6095960745587945e-05, 1.5740031813038513e-05, 1.555187554913573e-05, 1.5254239770001732e-05, 1.7964734070119448e-05, 1.740738116495777e-05, 1.65539349836763e-05, 1.579625313752331e-05, 1.4938361346139573e-05, 9.940669770003296e-06, 8.90496539796004e-06, 8.280208021460567e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 6.355251207423862e-05, "first_to_last_ratio": 1.1581492891728726, "imbalance_abs_log_ratio": 0.14682329067976, "last_quartile_mean": 5.487419684868655e-05, "mean": 5.5652203968747926e-05, "normalized": [ 1.073502673464316, 1.23547496070161, 1.1160500432949931, 1.0336067066310568, 1.3101082943647209, 1.211723985507878, 1.139081818888686, 1.0161201939778952, 0.9572468232645979, 0.9033875850836952, 0.8665209899887577, 0.8405715598096668, 0.9692344856036623, 0.9023117273289738, 0.8501998264784699, 0.8114601877423833, 0.9072991235737465, 0.8415878002188841, 0.8323677031831154, 0.8191080734001714, 1.1502760214254166, 1.1260468000697244, 1.109730081712391, 1.0888209973941456, 1.281544525919463, 1.2427871065715594, 1.1825361960317191, 1.1304250142500756, 1.0768475856924933, 0.7214608754949775, 0.6484744056315201, 0.604085827299234 ], "peak_layer": 5, "peak_normalized": 1.3101082943647209, "population_cv": 0.1823612898554525, "rest_mean": 5.409307979991556e-05, "spike_contrast": 1.184467120700765, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.40714744804427e-05, "top_five_layers": [ 5, 25, 26, 2, 6 ], "top_five_spike_overlap": 1 }, "values": [ 5.974278974463232e-05, 6.875690451124683e-05, 6.211064464878291e-05, 5.752249126089737e-05, 7.29104140191339e-05, 6.743511039530858e-05, 6.339241372188553e-05, 5.654932829202153e-05, 5.3272895456757396e-05, 5.027551014791243e-05, 4.822380287805572e-05, 4.6779659896856174e-05, 5.394003528635949e-05, 5.021563629270531e-05, 4.7315494157373905e-05, 4.51595478807576e-05, 5.0493195885792375e-05, 4.683621591539122e-05, 4.632309719454497e-05, 4.558516957331449e-05, 6.401539576472715e-05, 6.266698619583622e-05, 6.175892485771328e-05, 6.059528823243454e-05, 7.132077735150233e-05, 6.91638415446505e-05, 6.581074558198452e-05, 6.291064346441999e-05, 5.9928941482212394e-05, 4.015088779851794e-05, 3.608902989071794e-05, 3.36187076754868e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.056359002366662025, "first_to_last_ratio": 1.1581492634824053, "imbalance_abs_log_ratio": 0.14682326849741456, "last_quartile_mean": 0.048662986839190125, "mean": 0.04935293091693893, "normalized": [ 1.0735027068440102, 1.2354748993709166, 1.1160500154540713, 1.0336067265198758, 1.3101082243546494, 1.2117239797842965, 1.1390818117786266, 1.0161201880230026, 0.9572468112667539, 0.9033875963717879, 0.8665209614739626, 0.8405715340537224, 0.9692345137692789, 0.9023117420782842, 0.8501997997004039, 0.8114602136635408, 0.9072991075967483, 0.8415878325317785, 0.8323677016051542, 0.8191080408692867, 1.1502760406030972, 1.1260467874265008, 1.1097301546122915, 1.08882100605225, 1.2815445306324935, 1.2427871306888552, 1.1825362709464178, 1.1304250079124398, 1.0768475697716764, 0.7214608844330853, 0.6484743645515323, 0.6040858452592086 ], "peak_layer": 5, "peak_normalized": 1.3101082243546494, "population_cv": 0.18236129016539168, "rest_mean": 0.04797028384550854, "spike_contrast": 1.1844671439854952, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.05681922510266304, "top_five_layers": [ 5, 25, 26, 2, 6 ], "top_five_spike_overlap": 1 }, "values": [ 0.05298050493001938, 0.06097430735826492, 0.055080339312553406, 0.051011521369218826, 0.06465768069028854, 0.05980212986469269, 0.056217025965452194, 0.050148509442806244, 0.04724293574690819, 0.04458482563495636, 0.04276534914970398, 0.04148466885089874, 0.04783456400036812, 0.04453172907233238, 0.04195985198020935, 0.040047939866781235, 0.044777870178222656, 0.041534826159477234, 0.04107978567481041, 0.0404253825545311, 0.05676949396729469, 0.055573709309101105, 0.054768435657024384, 0.05373650789260864, 0.06324797868728638, 0.061335187405347824, 0.05836163088679314, 0.05578978732228279, 0.05314558371901512, 0.03560620918869972, 0.032004110515117645, 0.029813406988978386 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 2.571848341403893e-06, "first_to_last_ratio": 0.6936152846358217, "imbalance_abs_log_ratio": 0.36583781708643226, "last_quartile_mean": 3.7078887942243455e-06, "mean": 3.0139316891109047e-06, "normalized": [ 0.9610836871966651, 0.8596704662908717, 0.8155148405706824, 0.7901386390354868, 0.8862926494908768, 0.8549144465467095, 0.8380097283662274, 0.8209359471613547, 0.7674611842970476, 0.7529954699445681, 0.755441716097578, 0.7619060192528535, 0.7946208083653079, 0.7929300272396291, 0.7998289453362322, 0.8109816732022678, 0.8320606864588722, 0.8469268410315508, 0.8678545953127409, 0.9090025438100782, 1.346150865513476, 1.3948998604287037, 1.438810529593696, 1.4595695980549228, 1.618054150600037, 1.5801872016536944, 1.531617303400628, 1.4504302365409358, 1.2921003378110831, 0.8738920539378945, 0.7809298719068459, 0.7147870755504823 ], "peak_layer": 25, "peak_normalized": 1.618054150600037, "population_cv": 0.29451151220063904, "rest_mean": 2.761935185710037e-06, "spike_contrast": 1.5839302928287, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.374712807475589e-06, "top_five_layers": [ 25, 26, 27, 24, 28 ], "top_five_spike_overlap": 2 }, "values": [ 2.8966405807295814e-06, 2.590988060546806e-06, 2.457906020936207e-06, 2.381423882980016e-06, 2.6712255021266174e-06, 2.5766537419258384e-06, 2.5257040761061944e-06, 2.474244865879882e-06, 2.313075583515456e-06, 2.2694769086228916e-06, 2.276849727422814e-06, 2.296332695550518e-06, 2.394932835159125e-06, 2.389836936345091e-06, 2.4106298042170238e-06, 2.4442433641524985e-06, 2.5077740701817675e-06, 2.5525796445435844e-06, 2.6156544663535897e-06, 2.739671572271618e-06, 4.057206751895137e-06, 4.204132892482448e-06, 4.336476649768883e-06, 4.3990430640405975e-06, 4.87670467919088e-06, 4.762576281791553e-06, 4.616189926309744e-06, 4.371497652755352e-06, 3.894302153639728e-06, 2.633850954225636e-06, 2.3536692879133625e-06, 2.1543194179685088e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 4.7774661652510986e-05, "first_to_last_ratio": 1.3606902937746002, "imbalance_abs_log_ratio": 0.30799213993032315, "last_quartile_mean": 3.511060663186072e-05, "mean": 3.937230616202214e-05, "normalized": [ 1.153050320074888, 1.3046842593312258, 1.185163743383181, 1.1021442378698691, 1.4004355752633924, 1.2951074275884222, 1.1990825169319979, 1.0675939642870558, 1.0107703451263959, 0.9495584920432574, 0.9025323544739412, 0.870387145760331, 1.02451254335672, 0.9436735720878403, 0.8784366148580874, 0.8287596299249629, 0.9456936085153066, 0.8665660595641257, 0.8558110426375961, 0.8399143668032453, 1.1165514348271384, 1.0757345392267974, 1.044219864251227, 1.005544695133012, 1.207122746086348, 1.1626805586905793, 1.1066197892575365, 1.0581080535873695, 0.861270463199157, 0.6260932060378093, 0.568931497347916, 0.5432453324732679 ], "peak_layer": 5, "peak_normalized": 1.4004355752633924, "population_cv": 0.19807936889244088, "rest_mean": 3.871730661086514e-05, "spike_contrast": 1.1082719200882747, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.2909303738269954e-05, "top_five_layers": [ 5, 2, 6, 25, 7 ], "top_five_spike_overlap": 1 }, "values": [ 4.5398250222206116e-05, 5.136842810316011e-05, 4.666262975661084e-05, 4.339396036812104e-05, 5.5138378229457885e-05, 5.0991366151720285e-05, 4.7210643970174715e-05, 4.203363641863689e-05, 3.979635948780924e-05, 3.7386307667475194e-05, 3.553478018147871e-05, 3.4269149182364345e-05, 4.033742152387276e-05, 3.715460479725152e-05, 3.4586075344122946e-05, 3.263017788412981e-05, 3.723413828993216e-05, 3.4118704206775874e-05, 3.369525438756682e-05, 3.306936559965834e-05, 4.3961204937659204e-05, 4.235414962749928e-05, 4.111334419576451e-05, 3.959061359637417e-05, 4.752720633405261e-05, 4.577741492539644e-05, 4.3570173147600144e-05, 4.166015423834324e-05, 3.3910204365383834e-05, 2.4650733394082636e-05, 2.240014509879984e-05, 2.138882155122701e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 3.456847571214894e-05, "first_to_last_ratio": 1.5701697564204726, "imbalance_abs_log_ratio": 0.45118373862369976, "last_quartile_mean": 2.2015756940163556e-05, "mean": 2.7054974339080218e-05, "normalized": [ 1.226200509878072, 1.358310564673835, 1.2522492965328726, 1.1618003576472389, 1.4892989026707424, 1.3766247227039672, 1.2419860315617064, 1.1152286716367241, 1.065475516019289, 0.9818750736911386, 0.9226257153752625, 0.8877160786802075, 1.0589326610685907, 0.9720442519260525, 0.8995208638362313, 0.8354488090627477, 0.9675653182347209, 0.8800404807046276, 0.8695020979475038, 0.8617927489629916, 1.1100727007741817, 1.0464979019012217, 0.9879386912082332, 0.9213195782228187, 1.1482692957873417, 1.1085864461856236, 1.0724951880125122, 1.0301617412471589, 0.6666016550743451, 0.5307747536116468, 0.48825715542504755, 0.46478621973534695 ], "peak_layer": 5, "peak_normalized": 1.4892989026707424, "population_cv": 0.2357755294039874, "rest_mean": 2.684044024876871e-05, "spike_contrast": 1.051154830743011, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.8213458426762374e-05, "top_five_layers": [ 5, 6, 2, 3, 7 ], "top_five_spike_overlap": 0 }, "values": [ 3.3174823329318315e-05, 3.674905747175217e-05, 3.387957258382812e-05, 3.1432478863280267e-05, 4.029294359497726e-05, 3.724454654729925e-05, 3.360190021339804e-05, 3.0172483093338087e-05, 2.8826412744820118e-05, 2.656460492289625e-05, 2.4961615054053254e-05, 2.401713572908193e-05, 2.864939597202465e-05, 2.6298632292309776e-05, 2.433651388855651e-05, 2.2603046090807766e-05, 2.6177454856224358e-05, 2.380947262281552e-05, 2.3524356947746128e-05, 2.331578070879914e-05, 3.0032988433958963e-05, 2.8312973881838843e-05, 2.6728655939223245e-05, 2.492627754691057e-05, 3.106639633188024e-05, 2.9992777854204178e-05, 2.9016329790465534e-05, 2.7870999474544078e-05, 1.8034890672424808e-05, 1.436009733879473e-05, 1.3209784810896963e-05, 1.2574779248097911e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00013472975115291774, "first_to_last_ratio": 1.152249671406799, "imbalance_abs_log_ratio": 0.1417162674431732, "last_quartile_mean": 0.00011692756743286736, "mean": 0.0001186287368000194, "normalized": [ 1.077462238653238, 1.1979162850471081, 1.0955607549370945, 1.025460207935335, 1.2920089183563641, 1.20300024819894, 1.1645023374350898, 1.0298977122667987, 0.9577298463846478, 0.9196830461164149, 0.8860163416923722, 0.8617924134170397, 0.9860792736765284, 0.9186178610753019, 0.8700975692431515, 0.836758019595944, 0.925537485141251, 0.8715078798802998, 0.8532992079947943, 0.8345603049421083, 1.099638474881435, 1.0815159157268401, 1.0727239531021266, 1.0533559487097333, 1.3429697455700387, 1.2965920292570456, 1.2306188826213058, 1.170931229335786, 0.9972635019397272, 0.6686565234451283, 0.6032221333822645, 0.5750237100387469 ], "peak_layer": 25, "peak_normalized": 1.3429697455700387, "population_cv": 0.18804428505882276, "rest_mean": 0.00011577196333005472, "spike_contrast": 1.1579255432997178, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013405531353782862, "top_five_layers": [ 25, 26, 5, 27, 6 ], "top_five_spike_overlap": 1 }, "values": [ 0.00012781798432115465, 0.0001421072956873104, 0.00012996498844586313, 0.00012164904910605401, 0.00015326938591897488, 0.00014271039981395006, 0.00013814344129059464, 0.00012217546463944018, 0.00011361428187228739, 0.00010910083801718429, 0.00010510699939914048, 0.0001022333453875035, 0.00011697733862092718, 0.00010897447646129876, 0.00010321857553208247, 9.926354687195271e-05, 0.00010979534272337332, 0.000103385878901463, 0.00010122580715687945, 9.900283475872129e-05, 0.0001304487232118845, 0.00012829886691179127, 0.00012725588749162853, 0.00012495828559622169, 0.00015931480447761714, 0.00015381307457573712, 0.00014598676352761686, 0.00013890609261579812, 0.00011830410949187353, 7.932187872938812e-05, 7.155947969295084e-05, 6.821433635195717e-05 ] } } }, "pre_attention_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "ee695e78937de01b7128d3866b8fc5dd328ebdf2c159be37918d4f02952d5248", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.0626930519938469, 0.09030644595623016, 0.20229491591453552, 0.20300470292568207, 0.19049881398677826, 0.12695962190628052, 0.12132706493139267, 0.11585728824138641, 0.13427142798900604, 0.10485932976007462, 0.10419609397649765, 0.10852954536676407, 0.1286265105009079, 0.09096142649650574, 0.10080574452877045, 0.10566592961549759, 0.1287592649459839, 0.09650235623121262, 0.1065765842795372, 0.11446042358875275, 0.12395420670509338, 0.07221997529268265, 0.07677769660949707, 0.08416415005922318, 0.09094838052988052, 0.059439923614263535, 0.062182098627090454, 0.06896103918552399, 0.0776243805885315, 0.06285123527050018, 0.07231374084949493, 0.08016443997621536 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.0218872432687931e-05, "first_to_last_ratio": 1.7688536166553719, "imbalance_abs_log_ratio": 0.5703316625481881, "last_quartile_mean": 5.777115944738398e-06, "mean": 8.99577897683912e-06, "normalized": [ 1.015120510025447, 1.426193098117947, 0.9521782097540027, 0.8590644304949817, 0.5371202162298174, 1.1991288712115724, 1.4445378224133605, 1.6543611608543838, 1.0799401804680242, 0.8807907262327929, 0.7434683118695354, 0.6656672975078936, 0.6104315770188654, 1.1357691191960058, 0.8905528691383922, 0.8427143045851134, 0.9644658902154691, 1.4222655738011825, 1.3695142927488677, 1.5470795000960074, 0.9070386238965868, 1.772261908326347, 1.4259652133440908, 1.5167465606009558, 0.6778422993622337, 0.9048993985505263, 0.7379728412283049, 0.7325887348632067, 0.6863452626245364, 0.41937766871344045, 0.46873608013192986, 0.5098614463781797 ], "peak_layer": 22, "peak_normalized": 1.772261908326347, "population_cv": 0.3694895363955271, "rest_mean": 8.562697320047408e-06, "spike_contrast": 1.3236973701006176, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.133441992351436e-05, "top_five_layers": [ 22, 8, 20, 24, 7 ], "top_five_spike_overlap": 2 }, "values": [ 9.131799743045121e-06, 1.2829717888962477e-05, 8.565584721509367e-06, 7.727953743597027e-06, 4.831814749195473e-06, 1.0787098290165886e-05, 1.299474297411507e-05, 1.4882267350913025e-05, 9.714903171698097e-06, 7.923398698039819e-06, 6.688076609862037e-06, 5.988195880490821e-06, 5.491307547345059e-06, 1.0217127965006512e-05, 8.011216777958907e-06, 7.580871624668362e-06, 8.676121979078744e-06, 1.2794386748282705e-05, 1.231984788319096e-05, 1.3917185242462438e-05, 8.159518984030001e-06, 1.594287641637493e-05, 1.2827667887904681e-05, 1.364431682304712e-05, 6.097719506215071e-06, 8.140274985635187e-06, 6.638640570599819e-06, 6.590206339751603e-06, 6.174210284370929e-06, 3.7726288155681686e-06, 4.216646175336791e-06, 4.586600880429614e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 3.903993297171837e-05, "first_to_last_ratio": 1.7196638614087034, "imbalance_abs_log_ratio": 0.542128842313001, "last_quartile_mean": 2.2702072101310478e-05, "mean": 3.5427242607966036e-05, "normalized": [ 0.8559926218013937, 1.335002669257341, 0.9175954412349157, 0.8553532303790067, 0.5354676477103637, 1.164920443352911, 1.4711707074254223, 1.6802966288685095, 1.0429104094534816, 0.8761333232456624, 0.746454340416498, 0.6721306969362814, 0.6248480786318724, 1.1516533665685003, 0.9135248596909631, 0.8578348064030016, 1.0161926484224861, 1.5411496931220092, 1.3957314535038965, 1.5523829167767649, 0.9412693027168233, 1.7219868185431593, 1.4756066554964533, 1.5279246160311273, 0.6623954452359255, 0.9478384039264143, 0.7188645466289122, 0.6684933591747187, 0.6885569413773248, 0.43695921493613205, 0.47829944139428854, 0.5250592713374396 ], "peak_layer": 22, "peak_normalized": 1.7219868185431593, "population_cv": 0.37377743207166786, "rest_mean": 3.368319509415111e-05, "spike_contrast": 1.3313790172581852, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.484509918256663e-05, "top_five_layers": [ 22, 8, 20, 18, 24 ], "top_five_spike_overlap": 2 }, "values": [ 3.032545828318689e-05, 4.729546344606206e-05, 3.2507876312593e-05, 3.0302806408144534e-05, 1.8970142264151946e-05, 4.126991916564293e-05, 5.211952156969346e-05, 5.9528276324272156e-05, 3.6947440094081685e-05, 3.103898779954761e-05, 2.6444819013704546e-05, 2.3811737264622934e-05, 2.2136644474812783e-05, 4.0799903217703104e-05, 3.236366683267988e-05, 3.0390721803996712e-05, 3.600090349209495e-05, 5.459868407342583e-05, 4.9446916818851605e-05, 5.49966462131124e-05, 3.334657594677992e-05, 6.100524478824809e-05, 5.227667497820221e-05, 5.41301560588181e-05, 2.3466844140784815e-05, 3.3579301089048386e-05, 2.5467388695687987e-05, 2.368287641729694e-05, 2.4393673811573535e-05, 1.5480260117328726e-05, 1.6944830349530093e-05, 1.860140218923334e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.034621003083884716, "first_to_last_ratio": 1.719663879066529, "imbalance_abs_log_ratio": 0.5421288525811853, "last_quartile_mean": 0.02013242442626506, "mean": 0.03141723314183764, "normalized": [ 0.8559926003654894, 1.3350026594230198, 0.9175954393805157, 0.8553531860719467, 0.5354676436019856, 1.1649204731112548, 1.4711706910444502, 1.6802966939386133, 1.0429103841997542, 0.8761332316578393, 0.7464543213397783, 0.6721307247691818, 0.6248480387104585, 1.1516533824342199, 0.913524827792166, 0.8578347775494449, 1.016192649145841, 1.5411497090864597, 1.3957314247310733, 1.5523830057972738, 0.9412693015327339, 1.7219868616018248, 1.4756066906987344, 1.5279247124426574, 0.6623954420551171, 0.9478383422118749, 0.7188645283494981, 0.6684933852862758, 0.6885569429966502, 0.43695921667435883, 0.47829944063171637, 0.5250592713677921 ], "peak_layer": 22, "peak_normalized": 1.7219868616018248, "population_cv": 0.3737774508701596, "rest_mean": 0.029870594161804074, "spike_contrast": 1.3313790619160886, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0397690836340189, "top_five_layers": [ 22, 8, 20, 18, 24 ], "top_five_spike_overlap": 2 }, "values": [ 0.026892919093370438, 0.041942089796066284, 0.02882830984890461, 0.026872830465435982, 0.01682291179895401, 0.036598578095436096, 0.04622011259198189, 0.05279027298092842, 0.032765358686447144, 0.027525682002305984, 0.023451529443264008, 0.021116487681865692, 0.019630996510386467, 0.03618176281452179, 0.02870042249560356, 0.026950795203447342, 0.03192596137523651, 0.04841865971684456, 0.043850019574165344, 0.04877157881855965, 0.029572077095508575, 0.05410006269812584, 0.046359479427337646, 0.048003166913986206, 0.020810632035136223, 0.029778458178043365, 0.022584734484553337, 0.021002212539315224, 0.02163255400955677, 0.013728049583733082, 0.015026845037937164, 0.01649590954184532 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 3.0281307417112657e-06, "first_to_last_ratio": 0.718593517994872, "imbalance_abs_log_ratio": 0.33045942466782174, "last_quartile_mean": 4.213968907151866e-06, "mean": 3.0849275436395374e-06, "normalized": [ 3.514595604880055, 1.4197053400239985, 0.2919726034059554, 0.3554728553957208, 0.2931017042124606, 0.5782881864191829, 0.6250278410167188, 0.7745473436033328, 0.5607168372687695, 0.5930419889481441, 0.5174100161724419, 0.613098918727386, 0.45161224110939396, 0.7821869100112886, 0.7203206544854375, 0.774975715350475, 0.4684345293399507, 1.0456311122214443, 0.7996520506813799, 1.0596265301853298, 0.6248858120496261, 1.2850132341145772, 1.3678341795824485, 1.554957012055977, 0.9780220071735557, 1.5187025519805644, 1.6900253067811957, 1.8780623724191345, 1.7924148515034133, 1.3160964246574722, 0.8573268670268313, 0.8972403971963382 ], "peak_layer": 1, "peak_normalized": 3.514595604880055, "population_cv": 0.6317687795269208, "rest_mean": 2.9922983386583198e-06, "spike_contrast": 1.198117582134408, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.5851252505381125e-06, "top_five_layers": [ 1, 28, 29, 27, 24 ], "top_five_spike_overlap": 1 }, "values": [ 1.0842272786248941e-05, 4.379688107292168e-06, 9.007143262351747e-07, 1.0966080026264535e-06, 9.041975204127084e-07, 1.7839771544458927e-06, 1.9281656022940297e-06, 2.389422434134758e-06, 1.7297708154728753e-06, 1.829491566240904e-06, 1.5961724102453445e-06, 1.8913657413577312e-06, 1.3931910416431492e-06, 2.4129899429681245e-06, 2.2221370272745844e-06, 2.390743929936434e-06, 1.445086581952637e-06, 3.2256962185783777e-06, 2.4668686364748282e-06, 3.268871068939916e-06, 1.9277274532214506e-06, 3.964172719861381e-06, 4.219669335725484e-06, 4.79692971566692e-06, 3.0171270282153273e-06, 4.685087333200499e-06, 5.213605618337169e-06, 5.793686341348803e-06, 5.529469945031451e-06, 4.060062110511353e-06, 2.6447912659932626e-06, 2.7679216145770624e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 2.5805636823861278e-05, "first_to_last_ratio": 1.8030480755216127, "imbalance_abs_log_ratio": 0.5894786080485107, "last_quartile_mean": 1.4312228927337856e-05, "mean": 2.3648718553204162e-05, "normalized": [ 0.7403061158493456, 1.334930283291355, 0.949547151715, 0.8683678320814703, 0.6070238863568634, 1.2390938065699657, 1.5194785652601768, 1.4709048398944615, 1.0985934514013325, 0.9403817173664587, 0.7963835598564364, 0.7104853029728448, 0.6859121550714132, 1.2264228012591123, 0.967630809049241, 0.8490811153426846, 0.9892012695883166, 1.5214984067001962, 1.4084618955867951, 1.4872389422631778, 0.8987607538928858, 1.7462872220464742, 1.5769226712134314, 1.525477171285471, 0.6767684113278256, 0.9453014080879009, 0.6765984246719825, 0.6069106644756864, 0.7046162252409626, 0.4385593417953237, 0.367764130073765, 0.4250896684116431 ], "peak_layer": 22, "peak_normalized": 1.7462872220464742, "population_cv": 0.37777155435632503, "rest_mean": 2.240127822814328e-05, "spike_contrast": 1.356391184426236, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.0384896308532917e-05, "top_five_layers": [ 22, 23, 24, 18, 7 ], "top_five_spike_overlap": 3 }, "values": [ 1.750729097693693e-05, 3.1569390557706356e-05, 2.245557334390469e-05, 2.0535786461550742e-05, 1.4355337043525651e-05, 2.9302980692591518e-05, 3.5933720937464386e-05, 3.478501457720995e-05, 2.5980327336583287e-05, 2.2238822566578165e-05, 1.8833450667443685e-05, 1.6802066966192797e-05, 1.622094350750558e-05, 2.9003327654208988e-05, 2.288322866661474e-05, 2.007968032557983e-05, 2.3393342416966334e-05, 3.59814875992015e-05, 3.3308318961644545e-05, 3.517129516694695e-05, 2.125454011547845e-05, 4.129745502723381e-05, 3.7292200431693345e-05, 3.607558028306812e-05, 1.6004705685190856e-05, 2.235516694781836e-05, 1.600068571860902e-05, 1.4352659491123632e-05, 1.6663270798744634e-05, 1.0371366442996077e-05, 8.697150406078435e-06, 1.0052825928141829e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 1.654580614740553e-05, "first_to_last_ratio": 1.8202855601195826, "imbalance_abs_log_ratio": 0.5989933899456095, "last_quartile_mean": 9.089676098028576e-06, "mean": 1.5515209341288028e-05, "normalized": [ 0.6447966642672858, 1.312364564493414, 0.947441290079358, 0.8747355118780685, 0.6896078572086426, 1.2880686338512608, 1.5332674418368533, 1.241117533973562, 1.1625614653420886, 1.0198622358521119, 0.8513410914064126, 0.745369711014843, 0.7585884211681131, 1.2984187373523783, 1.0089072957595913, 0.8526853550925979, 0.9576014662150802, 1.5105721768241687, 1.411860007920583, 1.375984956140623, 0.8419205768437011, 1.7706914207091484, 1.725810763592485, 1.4895780513620291, 0.6742667673666394, 0.946033365554031, 0.65535627406591, 0.5611678350126037, 0.7200921958870289, 0.4544971526131696, 0.31795130597398674, 0.35748187334223036 ], "peak_layer": 22, "peak_normalized": 1.7706914207091484, "population_cv": 0.38243180619299966, "rest_mean": 1.465195023060746e-05, "spike_contrast": 1.3770732374461916, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.0176808538963088e-05, "top_five_layers": [ 22, 23, 7, 18, 24 ], "top_five_spike_overlap": 3 }, "values": [ 1.0004155228671152e-05, 2.0361610950203612e-05, 1.4699749954161234e-05, 1.3571704585046973e-05, 1.0699410267989151e-05, 1.998465450014919e-05, 2.3788965336279944e-05, 1.925619835674297e-05, 1.803738450689707e-05, 1.582337608851958e-05, 1.3208735254011117e-05, 1.156456710305065e-05, 1.1769658158300444e-05, 2.0145238522673026e-05, 1.5653407899662852e-05, 1.3229591786512174e-05, 1.4857387213851325e-05, 2.3436843548552133e-05, 2.1905303583480418e-05, 2.134869464498479e-05, 1.3062573998467997e-05, 2.747264807112515e-05, 2.677631528058555e-05, 2.3111115297069773e-05, 1.0461390047566965e-05, 1.4677905710414052e-05, 1.0167989785259124e-05, 8.706636435817927e-06, 1.117238116421504e-05, 7.051618467812659e-06, 4.933081072522327e-06, 5.546406100620516e-06 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 7.788295988575555e-05, "first_to_last_ratio": 1.8027437652565572, "imbalance_abs_log_ratio": 0.5893098183461566, "last_quartile_mean": 4.320245693634206e-05, "mean": 7.014507190206132e-05, "normalized": [ 0.8303967903602477, 1.3250675796407025, 0.93378078635336, 0.9071588683922475, 0.5321785742459074, 1.22036723258259, 1.4747571455647606, 1.6587941336436314, 1.0304887389226507, 0.8417563798588832, 0.7497830071868703, 0.6960908100906597, 0.6246216240482398, 1.1323779729267014, 0.9152938878930372, 0.8552680511290085, 0.9733474541279368, 1.5046826737121695, 1.4181502637239793, 1.6259707764348845, 0.9477856289830491, 1.7058904685045664, 1.5403766834345194, 1.6284022476389088, 0.6993378370449352, 0.9581106425766917, 0.6925425591111084, 0.6303274543451968, 0.695568802546858, 0.3982996651257911, 0.38119443998594965, 0.47183081986395753 ], "peak_layer": 22, "peak_normalized": 1.7058904685045664, "population_cv": 0.3909807216741981, "rest_mean": 6.61915063473215e-05, "spike_contrast": 1.3822668639321394, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.149432589765638e-05, "top_five_layers": [ 22, 8, 24, 20, 23 ], "top_five_spike_overlap": 3 }, "values": [ 5.8248242567060515e-05, 9.294696064898744e-05, 6.55001203995198e-05, 6.363272404996678e-05, 3.7329704355215654e-05, 8.560274727642536e-05, 0.00010344694601371884, 0.00011635623377515003, 7.228370668599382e-05, 5.90450617892202e-05, 5.2593582950066775e-05, 4.882733992417343e-05, 4.38141287304461e-05, 7.943073433125392e-05, 6.420335557777435e-05, 5.999283894198015e-05, 6.827552715549245e-05, 0.00010554607433732599, 9.947625221684575e-05, 0.00011405383702367544, 6.648249109275639e-05, 0.00011965980957029387, 0.0001080498332157731, 0.00011422439274610952, 4.905510286334902e-05, 6.720673991367221e-05, 4.857844760408625e-05, 4.42143646068871e-05, 4.879072366748005e-05, 2.7938758648815565e-05, 2.6738911401480436e-05, 3.309660678496584e-05 ] } } }, "pre_mlp_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "e53fa425d89dfeb0e9395eca9bcd73dc0a55314abf45c5b10b43e2cbfa724345", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.059514161199331284, 0.08157797157764435, 0.14292451739311218, 0.10997853428125381, 0.0991024449467659, 0.1065012738108635, 0.12155740708112717, 0.1095810979604721, 0.06964563578367233, 0.08311934769153595, 0.09363401681184769, 0.0941167026758194, 0.08564268052577972, 0.09493856877088547, 0.09063401818275452, 0.09473779052495956, 0.10505300015211105, 0.09355910867452621, 0.09618847072124481, 0.11416846513748169, 0.047205954790115356, 0.053352247923612595, 0.06512638926506042, 0.07829347252845764, 0.05368548259139061, 0.05310077592730522, 0.061235878616571426, 0.07320034503936768, 0.05304349213838577, 0.0639304369688034, 0.07404901832342148, 0.08157160878181458 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 4.82012529801068e-06, "first_to_last_ratio": 0.1865964901159196, "imbalance_abs_log_ratio": 1.678806800413967, "last_quartile_mean": 2.58318111718836e-05, "mean": 1.2174142867138471e-05, "normalized": [ 0.84731309293166, 0.30216740247598767, 0.20847497110484697, 0.3591610252167397, 0.2904573847189895, 0.23159376898336892, 0.3156559904350187, 0.6126274779339936, 0.6629366097151811, 0.49942529183431916, 0.4518014687984496, 0.4623534707384525, 0.5116260417428511, 0.5352105075465564, 0.5943310035871131, 0.6020826478739598, 0.4436022543772791, 0.5566617125948595, 0.510879045619816, 0.4204925147325724, 0.9794341380867863, 1.3943543635581324, 1.5587966064004044, 1.673691468233419, 2.001742222871551, 2.5630912446222793, 3.050158416077702, 2.250568058359819, 2.5542856701618435, 2.0221305328963037, 1.315793590376243, 1.2171000053935013 ], "peak_layer": 27, "peak_normalized": 3.050158416077702, "population_cv": 0.7869133023910566, "rest_mean": 1.0998202368581486e-05, "spike_contrast": 1.6842953910599288, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.8524221559346188e-05, "top_five_layers": [ 27, 26, 29, 28, 30 ], "top_five_spike_overlap": 0 }, "values": [ 1.0315310646547005e-05, 3.678629127534805e-06, 2.5380040824529715e-06, 4.372477633296512e-06, 3.536069698384381e-06, 2.8194556307425955e-06, 3.842841124424012e-06, 7.458214440703159e-06, 8.070684998529032e-06, 6.080074854253326e-06, 5.50029562873533e-06, 5.628757207887247e-06, 6.2286085267260205e-06, 6.515729182865471e-06, 7.235470548039302e-06, 7.329840173042612e-06, 5.400477220973698e-06, 6.776879217795795e-06, 6.219514489202993e-06, 5.119135948916664e-06, 1.1923771126021165e-05, 1.697506922937464e-05, 1.897701258712914e-05, 2.0375759049784392e-05, 2.4369495804421604e-05, 3.1203438993543386e-05, 3.713306432473473e-05, 2.739873707469087e-05, 3.1096238672034815e-05, 2.4617706003482454e-05, 1.6018659152905457e-05, 1.4817149349255487e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 1.9254757148701174e-05, "first_to_last_ratio": 0.18641203271324644, "imbalance_abs_log_ratio": 1.6797958256314607, "last_quartile_mean": 0.00010329138558518025, "mean": 4.860563856823319e-05, "normalized": [ 0.8580756181007752, 0.307854345179309, 0.20839115571457204, 0.36395941092193485, 0.28660008595587366, 0.2303312178879652, 0.31572593197310506, 0.5982019005084565, 0.6646135081041679, 0.49679759548281455, 0.45269614900860733, 0.4636355770906204, 0.5066917499190732, 0.5306410569961122, 0.5933931028406856, 0.5953662154288047, 0.44149737660075283, 0.5564627328048252, 0.5148415611559383, 0.4160227281994753, 0.9874566267855747, 1.3938065226762122, 1.5533395837896518, 1.6628734539799683, 1.9986590146469976, 2.5625638443241874, 3.0604462700115467, 2.262330835504778, 2.5642559818476, 2.0165454651300108, 1.3231406202808387, 1.212782761148765 ], "peak_layer": 27, "peak_normalized": 3.0604462700115467, "population_cv": 0.7886048391479346, "rest_mean": 4.393205303872323e-05, "spike_contrast": 1.6808456541400234, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 7.384300042758695e-05, "top_five_layers": [ 27, 29, 26, 28, 30 ], "top_five_spike_overlap": 0 }, "values": [ 4.170731335761957e-05, 1.4963457033445593e-05, 1.012898519547889e-05, 1.769047958077863e-05, 1.393038019159576e-05, 1.1195395927643403e-05, 1.5346060536103323e-05, 2.9075985366944224e-05, 3.23039639624767e-05, 2.4147164367605e-05, 2.2003585399943404e-05, 2.253530328744091e-05, 2.462807606207207e-05, 2.5792147425818257e-05, 2.8842250685556792e-05, 2.8938155082869343e-05, 2.1459261915879324e-05, 2.7047226467402652e-05, 2.502420284145046e-05, 2.022105036303401e-05, 4.7995959903346375e-05, 6.774685607524589e-05, 7.550106238340959e-05, 8.082502608885989e-05, 9.714609768707305e-05, 0.00012455505202524364, 0.00014875494525767863, 0.00010996203491231427, 0.00012463729945011437, 9.801548003451899e-05, 6.431209476431832e-05, 5.894808055018075e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.0170753113925457, "first_to_last_ratio": 0.1864120329900063, "imbalance_abs_log_ratio": 1.6797958241467934, "last_quartile_mean": 0.09159983461722732, "mean": 0.04310396651271731, "normalized": [ 0.85807561958318, 0.30785433509656246, 0.2083911502107681, 0.3639594336905506, 0.28660009585203844, 0.23033122056447144, 0.31572594425899664, 0.5982018923322023, 0.6646134712169605, 0.49679758631829163, 0.45269615261673796, 0.4636355714908454, 0.5066917728271525, 0.5306410677572783, 0.5933930801910838, 0.5953662220395015, 0.4414973702294748, 0.5564627321744956, 0.5148415430915833, 0.41602274819382246, 0.9874566147395789, 1.3938064694186876, 1.5533395749070587, 1.6628734275725783, 1.9986590351084754, 2.5625638839500278, 3.060446242389404, 2.2623306963494136, 2.564256099074183, 2.016545524263983, 1.3231406720828875, 1.2127827504077247 ], "peak_layer": 27, "peak_normalized": 3.060446242389404, "population_cv": 0.7886048413669244, "rest_mean": 0.038959384240486006, "spike_contrast": 1.680845630889505, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.06548471078276634, "top_five_layers": [ 27, 29, 26, 28, 30 ], "top_five_spike_overlap": 0 }, "values": [ 0.03698646277189255, 0.013269742950797081, 0.00898248516023159, 0.01568809524178505, 0.012353600934147835, 0.009928189218044281, 0.013609040528535843, 0.025784874334931374, 0.02864747680723667, 0.02141394652426243, 0.019512999802827835, 0.01998453214764595, 0.021840425208210945, 0.02287273481488228, 0.025577595457434654, 0.02566264569759369, 0.019030287861824036, 0.02398575097322464, 0.022191712632775307, 0.017932230606675148, 0.042563296854496, 0.060078587383031845, 0.0669550970196724, 0.07167644053697586, 0.08615013211965561, 0.1104566678404808, 0.13191737234592438, 0.09751542657613754, 0.11052960902452469, 0.08692111074924469, 0.05703261122107506, 0.05227574706077576 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 1.1588433679321497e-06, "first_to_last_ratio": 0.16760061723686692, "imbalance_abs_log_ratio": 1.7861714081434006, "last_quartile_mean": 6.914314440109592e-06, "mean": 3.171414723723842e-06, "normalized": [ 0.9043688239494265, 0.28150648527065036, 0.2049610848619364, 0.389426048691104, 0.29231514468897274, 0.21681967753721187, 0.25945774348520473, 0.37436625113416067, 0.539362174497038, 0.45200474501838667, 0.4150965924050211, 0.45567300512231684, 0.48157135879382684, 0.5487738265420559, 0.5953623429930616, 0.6003830163574333, 0.4487372583725034, 0.5518622914840494, 0.4973622632091661, 0.36887837915809624, 1.01276176965076, 1.4096107306815575, 1.5869930632131166, 1.6707558630603627, 2.011405559370987, 2.5536791487107977, 3.2958852093584627, 2.361796990195475, 2.5722931042324078, 2.004401989420581, 1.385304071382516, 1.2568239871513527 ], "peak_layer": 27, "peak_normalized": 3.2958852093584627, "population_cv": 0.8273988992696941, "rest_mean": 2.8552684899228487e-06, "spike_contrast": 1.708632446814496, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.878604386249208e-06, "top_five_layers": [ 27, 29, 26, 28, 25 ], "top_five_spike_overlap": 1 }, "values": [ 2.8681286039500264e-06, 8.927738122110895e-07, 6.50016602321557e-07, 1.235031504620565e-06, 9.270525538340735e-07, 6.876251177345694e-07, 8.22848107873142e-07, 1.1872706409121747e-06, 1.7105411416196148e-06, 1.4334945035443525e-06, 1.3164434449208784e-06, 1.4451280776484055e-06, 1.5272624978024396e-06, 1.7403893934897496e-06, 1.8881409005189198e-06, 1.9040635379496962e-06, 1.4231319482860272e-06, 1.7501841966804932e-06, 1.5773420045661624e-06, 1.1698663229253725e-06, 3.211887587895035e-06, 4.470460226002615e-06, 5.0330131671216805e-06, 5.29865974385757e-06, 6.379001206369139e-06, 8.09877565188799e-06, 1.0452618880663067e-05, 7.490237749152584e-06, 8.157808224495966e-06, 6.356789981509792e-06, 4.393373728817096e-06, 3.985910097981105e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 1.2788041033218178e-05, "first_to_last_ratio": 0.20740197905116964, "imbalance_abs_log_ratio": 1.5730964410375903, "last_quartile_mean": 6.165824015624821e-05, "mean": 3.065003527069621e-05, "normalized": [ 0.9304826915811536, 0.329920143883825, 0.16666420785934835, 0.3427765562826807, 0.3213800429721008, 0.25543172331979347, 0.3301190752203418, 0.6610464573629266, 0.7430131116135551, 0.5518001566512873, 0.5022888057404364, 0.5283558137769104, 0.5830541568544981, 0.5917045245989336, 0.6564716300936152, 0.6561907404707829, 0.4850305330711591, 0.6106102419090336, 0.5509982592112471, 0.4600163121600251, 0.9999802323297369, 1.4132753954841426, 1.5860599615192474, 1.6498429741075682, 2.0865385633138502, 2.556001799423182, 2.908934002215039, 2.1592942575155276, 2.2676280378720706, 1.8257220144113655, 1.1941851691925098, 1.0951824079821058 ], "peak_layer": 27, "peak_normalized": 2.908934002215039, "population_cv": 0.7340885103409986, "rest_mean": 2.7544508847549202e-05, "spike_contrast": 1.7215728266619377, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.741987795569003e-05, "top_five_layers": [ 27, 26, 29, 28, 25 ], "top_five_spike_overlap": 1 }, "values": [ 2.85193273157347e-05, 1.0112064046552405e-05, 5.108263849251671e-06, 1.0506113540031947e-05, 9.850309652392752e-06, 7.828991329006385e-06, 1.011816129903309e-05, 2.0261097233742476e-05, 2.2773378077545203e-05, 1.691269426373765e-05, 1.5395169612020254e-05, 1.61941243277397e-05, 1.7870630472316407e-05, 1.8135764548787847e-05, 2.012087861658074e-05, 2.0112269339733757e-05, 1.486620294599561e-05, 1.8715225451160222e-05, 1.6888116078916937e-05, 1.4099516192800365e-05, 3.0649429390905425e-05, 4.3316940718796104e-05, 4.8612793762004e-05, 5.05677453475073e-05, 6.39524805592373e-05, 7.83415453042835e-05, 8.915892976801842e-05, 6.61824451526627e-05, 6.95028793415986e-05, 5.5958444136194885e-05, 3.6601817555492744e-05, 3.3567379432497546e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 8.205029416785692e-06, "first_to_last_ratio": 0.2391065666280881, "imbalance_abs_log_ratio": 1.4308459409548424, "last_quartile_mean": 3.431536629250331e-05, "mean": 1.829659461805022e-05, "normalized": [ 0.9872440986182439, 0.3539593504441842, 0.15117892055423085, 0.32563512328066496, 0.3682364023064313, 0.29421281920877523, 0.35428315105358854, 0.752815837581659, 0.8270407397563934, 0.6031910754482651, 0.5353095634128164, 0.5927777094882549, 0.6680616494252488, 0.658902476461229, 0.7187848607923553, 0.7356003722371095, 0.5368016111702437, 0.6797427290698828, 0.5954201590730455, 0.515259276920935, 1.029752149513378, 1.4358053711348797, 1.6326916506779163, 1.6432478289116168, 2.1518305201935983, 2.568767001413535, 2.6641144999889734, 1.9550681140184822, 1.9770708475991239, 1.63427764733798, 1.0602400099629332, 0.9926764329440259 ], "peak_layer": 27, "peak_normalized": 2.6641144999889734, "population_cv": 0.672292119478803, "rest_mean": 1.63359264425472e-05, "spike_contrast": 1.768139864448528, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.8884202765766532e-05, "top_five_layers": [ 27, 26, 25, 29, 28 ], "top_five_spike_overlap": 1 }, "values": [ 1.8063205061480403e-05, 6.476250746345613e-06, 2.7660594241751824e-06, 5.958013844065135e-06, 6.7374721766100265e-06, 5.38309268449666e-06, 6.482175194832962e-06, 1.3773966202279553e-05, 1.51320291479351e-05, 1.1036342584702652e-05, 9.794342076929752e-06, 1.0845813449122943e-05, 1.2223253179399762e-05, 1.2055671504640486e-05, 1.3151315215509385e-05, 1.3458981811709236e-05, 9.821641469898168e-06, 1.2436977158358786e-05, 1.089416127797449e-05, 9.427490113012027e-06, 1.884095763671212e-05, 2.627034882607404e-05, 2.9872697268729098e-05, 3.0065839382587e-05, 3.93711707147304e-05, 4.699968849308789e-05, 4.8744223022367805e-05, 3.577108873287216e-05, 3.617366382968612e-05, 2.9901715606683865e-05, 1.9398781660129316e-05, 1.8162598280468956e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 3.888845799338014e-05, "first_to_last_ratio": 0.17419551219851626, "imbalance_abs_log_ratio": 1.7475769772462426, "last_quartile_mean": 0.00022324603833112633, "mean": 0.00010316907963670019, "normalized": [ 0.8317983927844682, 0.30223664517787197, 0.1629363759005858, 0.33224426112343525, 0.2755058487774388, 0.22910253381025833, 0.31154631493745594, 0.570142269867155, 0.6474387781261639, 0.5061136242927232, 0.46616951598236733, 0.46757623482741634, 0.5102410409847279, 0.5375089174425381, 0.6089452549592864, 0.5987897837253249, 0.4549892871128313, 0.5645214243398781, 0.529224569515293, 0.4167078314047592, 0.9408506454011253, 1.3270871263823116, 1.489850699392662, 1.6073915039029665, 2.0912572277111083, 2.7522124677211472, 3.3156901633092493, 2.4499550530498873, 2.422065678593832, 1.9013164376105216, 1.2368341215280592, 1.1417499703051508 ], "peak_layer": 27, "peak_normalized": 3.3156901633092493, "population_cv": 0.8206056335416554, "rest_mean": 9.378284388124787e-05, "spike_contrast": 1.6405426232431233, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00015385475271614268, "top_five_layers": [ 27, 26, 28, 29, 25 ], "top_five_spike_overlap": 1 }, "values": [ 8.581587462686002e-05, 3.1181476515484974e-05, 1.6809995941002853e-05, 3.42773346346803e-05, 2.8423684852896258e-05, 2.363629755564034e-05, 3.214194657630287e-05, 5.882105324417353e-05, 6.679566286038607e-05, 5.221527680987492e-05, 4.8094279918586835e-05, 4.8239409807138145e-05, 5.26410985912662e-05, 5.545430030906573e-05, 6.282432150328532e-05, 6.177659088280052e-05, 4.6940825995989144e-05, 5.82411557843443e-05, 5.459961175802164e-05, 4.299136344343424e-05, 9.706669516162947e-05, 0.00013691435742657632, 0.00015370652545243502, 0.00016583310207352042, 0.0002157530834665522, 0.0002839432272594422, 0.0003420767025090754, 0.00025275960797443986, 0.00024988228688016534, 0.000196157066966407, 0.00012760303798131645, 0.00011779329361161217 ] } } } }, "selector": { "expected_uniform_count": 16, "identities_unique": true, "identity_order_sha256": "97fd9721a263024f6a746638f32e0ad9a988cd4c111c00e9e178c1022e5c9d5e", "passed": true, "uniform_count": 16, "uniform_indices": [ 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55 ], "visit_count": 65, "visits": [ { "branch": "attention", "coefficient": "detached_learned", "group": 1, "index": 0, "kind": "depth", "layer": 1, "offset": 1, "sources": 1, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 1, "index": 1, "kind": "depth", "layer": 1, "offset": 1, "sources": 2, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 1, "index": 2, "kind": "depth", "layer": 2, "offset": 2, "sources": 2, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 1, "index": 3, "kind": "depth", "layer": 2, "offset": 2, "sources": 2, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 1, "index": 4, "kind": "depth", "layer": 3, "offset": 3, "sources": 2, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 1, "index": 5, "kind": "depth", "layer": 3, "offset": 3, "sources": 2, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 1, "index": 6, "kind": "depth", "layer": 4, "offset": 4, "sources": 2, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 1, "index": 7, "kind": "depth", "layer": 4, "offset": 4, "sources": 2, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 2, "index": 8, "kind": "depth", "layer": 5, "offset": 1, "sources": 2, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 2, "index": 9, "kind": "depth", "layer": 5, "offset": 1, "sources": 3, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 2, "index": 10, "kind": "depth", "layer": 6, "offset": 2, "sources": 3, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 2, "index": 11, "kind": "depth", "layer": 6, "offset": 2, "sources": 3, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 2, "index": 12, "kind": "depth", "layer": 7, "offset": 3, "sources": 3, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 2, "index": 13, "kind": "depth", "layer": 7, "offset": 3, "sources": 3, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 2, "index": 14, "kind": "depth", "layer": 8, "offset": 4, "sources": 3, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 2, "index": 15, "kind": "depth", "layer": 8, "offset": 4, "sources": 3, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 3, "index": 16, "kind": "depth", "layer": 9, "offset": 1, "sources": 3, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 3, "index": 17, "kind": "depth", "layer": 9, "offset": 1, "sources": 4, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 3, "index": 18, "kind": "depth", "layer": 10, "offset": 2, "sources": 4, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 3, "index": 19, "kind": "depth", "layer": 10, "offset": 2, "sources": 4, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 3, "index": 20, "kind": "depth", "layer": 11, "offset": 3, "sources": 4, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 3, "index": 21, "kind": "depth", "layer": 11, "offset": 3, "sources": 4, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 3, "index": 22, "kind": "depth", "layer": 12, "offset": 4, "sources": 4, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 3, "index": 23, "kind": "depth", "layer": 12, "offset": 4, "sources": 4, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 4, "index": 24, "kind": "depth", "layer": 13, "offset": 1, "sources": 4, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 4, "index": 25, "kind": "depth", "layer": 13, "offset": 1, "sources": 5, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 4, "index": 26, "kind": "depth", "layer": 14, "offset": 2, "sources": 5, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 4, "index": 27, "kind": "depth", "layer": 14, "offset": 2, "sources": 5, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 4, "index": 28, "kind": "depth", "layer": 15, "offset": 3, "sources": 5, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 4, "index": 29, "kind": "depth", "layer": 15, "offset": 3, "sources": 5, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 4, "index": 30, "kind": "depth", "layer": 16, "offset": 4, "sources": 5, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 4, "index": 31, "kind": "depth", "layer": 16, "offset": 4, "sources": 5, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 5, "index": 32, "kind": "depth", "layer": 17, "offset": 1, "sources": 5, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 5, "index": 33, "kind": "depth", "layer": 17, "offset": 1, "sources": 6, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 5, "index": 34, "kind": "depth", "layer": 18, "offset": 2, "sources": 6, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 5, "index": 35, "kind": "depth", "layer": 18, "offset": 2, "sources": 6, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 5, "index": 36, "kind": "depth", "layer": 19, "offset": 3, "sources": 6, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 5, "index": 37, "kind": "depth", "layer": 19, "offset": 3, "sources": 6, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 5, "index": 38, "kind": "depth", "layer": 20, "offset": 4, "sources": 6, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 5, "index": 39, "kind": "depth", "layer": 20, "offset": 4, "sources": 6, "use_uniform": false }, { "branch": "attention", "coefficient": "uniform", "group": 6, "index": 40, "kind": "depth", "layer": 21, "offset": 1, "sources": 6, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 6, "index": 41, "kind": "depth", "layer": 21, "offset": 1, "sources": 7, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 6, "index": 42, "kind": "depth", "layer": 22, "offset": 2, "sources": 7, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 6, "index": 43, "kind": "depth", "layer": 22, "offset": 2, "sources": 7, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 6, "index": 44, "kind": "depth", "layer": 23, "offset": 3, "sources": 7, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 6, "index": 45, "kind": "depth", "layer": 23, "offset": 3, "sources": 7, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 6, "index": 46, "kind": "depth", "layer": 24, "offset": 4, "sources": 7, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 6, "index": 47, "kind": "depth", "layer": 24, "offset": 4, "sources": 7, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 7, "index": 48, "kind": "depth", "layer": 25, "offset": 1, "sources": 7, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 7, "index": 49, "kind": "depth", "layer": 25, "offset": 1, "sources": 8, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 7, "index": 50, "kind": "depth", "layer": 26, "offset": 2, "sources": 8, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 7, "index": 51, "kind": "depth", "layer": 26, "offset": 2, "sources": 8, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 7, "index": 52, "kind": "depth", "layer": 27, "offset": 3, "sources": 8, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 7, "index": 53, "kind": "depth", "layer": 27, "offset": 3, "sources": 8, "use_uniform": true }, { "branch": "attention", "coefficient": "uniform", "group": 7, "index": 54, "kind": "depth", "layer": 28, "offset": 4, "sources": 8, "use_uniform": true }, { "branch": "mlp", "coefficient": "uniform", "group": 7, "index": 55, "kind": "depth", "layer": 28, "offset": 4, "sources": 8, "use_uniform": true }, { "branch": "attention", "coefficient": "detached_learned", "group": 8, "index": 56, "kind": "depth", "layer": 29, "offset": 1, "sources": 8, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 8, "index": 57, "kind": "depth", "layer": 29, "offset": 1, "sources": 9, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 8, "index": 58, "kind": "depth", "layer": 30, "offset": 2, "sources": 9, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 8, "index": 59, "kind": "depth", "layer": 30, "offset": 2, "sources": 9, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 8, "index": 60, "kind": "depth", "layer": 31, "offset": 3, "sources": 9, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 8, "index": 61, "kind": "depth", "layer": 31, "offset": 3, "sources": 9, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 8, "index": 62, "kind": "depth", "layer": 32, "offset": 4, "sources": 9, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 8, "index": 63, "kind": "depth", "layer": 32, "offset": 4, "sources": 9, "use_uniform": false }, { "branch": "output", "coefficient": "detached_learned", "group": null, "index": 64, "kind": "output", "layer": null, "offset": null, "sources": 9, "use_uniform": false } ], "visits_sha256": "ecae65dd230eef0be68361d7f87960e32badc2e7c90cab749f34e981d307b2d6" } }, "uniform_output_only": { "bits_per_byte": 1.8740129107275643, "forward": { "activation_sha256": { "attention_branch_output": "5e6ecbcd78543fc42d4098a51169dca446d404ce384015643e84d2c78433c949", "mlp_branch_output": "c0b8018c349b1bd09e5d17858eb312a2323da0e92de9651e2bb08ae4adf115c7", "post_attention_state": "90a633f12d984935028fe5d7212be48e7da7b4a0806751939a551938d16dc540", "post_mlp_state": "89f2226456bfe96e245d2aad0f699d32c676f9f53099789cd870253d002402f5", "pre_attention_input": "ee695e78937de01b7128d3866b8fc5dd328ebdf2c159be37918d4f02952d5248", "pre_mlp_input": "e53fa425d89dfeb0e9395eca9bcd73dc0a55314abf45c5b10b43e2cbfa724345" }, "logits_sha256": "bdacba5675f0579cec629bbf01b36c8660ad576cfdf9fef1bea8259eaeac9ec2", "loss_nats": 1.2989667654037476, "mixer_summary_sha256": "eb3ac0f07de9b755d6e67a510ab68d09d9ad666bb7b5d020ba7d8c5055c399b2" }, "loss_nats": 1.2989667654037476, "loss_scale": 1.0, "mixers": [ { "branch": "attention", "entropy_mean": -0.0, "group": 1, "latest_source_mass": 1.0, "layer": 1, "max_source_mass": 1.0, "mixer_index": 0, "normalized_entropy": 1.0, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 1.0, "median": 1.0, "p05": 1.0, "p95": 1.0 } ], "sources": 1 }, { "branch": "mlp", "entropy_mean": 0.22886337339878082, "group": 1, "latest_source_mass": 0.06350618600845337, "layer": 1, "max_source_mass": 0.9364938139915466, "mixer_index": 1, "normalized_entropy": 0.3301800538435402, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.9364938139915466, "median": 0.9452826976776123, "p05": 0.8725647330284119, "p95": 0.9712384939193726 }, { "label": "current_group_partial", "mean": 0.06350618600845337, "median": 0.0547172985970974, "p05": 0.028761589899659157, "p95": 0.12743529677391052 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.603729784488678, "group": 1, "latest_source_mass": 0.4133155345916748, "layer": 2, "max_source_mass": 0.5866844654083252, "mixer_index": 2, "normalized_entropy": 0.8709979661187783, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.5866844654083252, "median": 0.5734395384788513, "p05": 0.29813364148139954, "p95": 0.8843642473220825 }, { "label": "current_group_partial", "mean": 0.4133155345916748, "median": 0.4265604615211487, "p05": 0.11563576757907867, "p95": 0.7018663287162781 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.3967982828617096, "group": 1, "latest_source_mass": 0.14110636711120605, "layer": 2, "max_source_mass": 0.858893632888794, "mixer_index": 3, "normalized_entropy": 0.5724589149178446, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.858893632888794, "median": 0.8705973029136658, "p05": 0.7599387168884277, "p95": 0.9270997643470764 }, { "label": "current_group_partial", "mean": 0.14110636711120605, "median": 0.12940272688865662, "p05": 0.07290027290582657, "p95": 0.2400612235069275 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6852496862411499, "group": 1, "latest_source_mass": 0.518975019454956, "layer": 3, "max_source_mass": 0.518975019454956, "mixer_index": 4, "normalized_entropy": 0.9886063241108252, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.48102498054504395, "median": 0.48102718591690063, "p05": 0.38786420226097107, "p95": 0.580143928527832 }, { "label": "current_group_partial", "mean": 0.518975019454956, "median": 0.5189728140830994, "p05": 0.41985607147216797, "p95": 0.6121357679367065 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.5890734195709229, "group": 1, "latest_source_mass": 0.3123423159122467, "layer": 3, "max_source_mass": 0.6876576542854309, "mixer_index": 5, "normalized_entropy": 0.8498533011344741, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.6876576542854309, "median": 0.6909762620925903, "p05": 0.48803940415382385, "p95": 0.8900076746940613 }, { "label": "current_group_partial", "mean": 0.3123423159122467, "median": 0.3090237081050873, "p05": 0.10999235510826111, "p95": 0.5119606256484985 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6873323321342468, "group": 1, "latest_source_mass": 0.4831485152244568, "layer": 4, "max_source_mass": 0.5168514251708984, "mixer_index": 6, "normalized_entropy": 0.9916109470127238, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.5168514251708984, "median": 0.5219584703445435, "p05": 0.4251725673675537, "p95": 0.5947866439819336 }, { "label": "current_group_partial", "mean": 0.4831485152244568, "median": 0.47804147005081177, "p05": 0.405213326215744, "p95": 0.5748274326324463 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.487301766872406, "group": 1, "latest_source_mass": 0.2060384750366211, "layer": 4, "max_source_mass": 0.7939615249633789, "mixer_index": 7, "normalized_entropy": 0.7030278424832499, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.7939615249633789, "median": 0.7953404188156128, "p05": 0.6590759754180908, "p95": 0.9329437017440796 }, { "label": "current_group_partial", "mean": 0.2060384750366211, "median": 0.20465964078903198, "p05": 0.06705628335475922, "p95": 0.3409239649772644 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.674697756767273, "group": 2, "latest_source_mass": 0.42952626943588257, "layer": 5, "max_source_mass": 0.5704737305641174, "mixer_index": 8, "normalized_entropy": 0.9733831077870527, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.5704737305641174, "median": 0.5696132779121399, "p05": 0.4695202708244324, "p95": 0.6750621199607849 }, { "label": "completed_group_1", "mean": 0.42952626943588257, "median": 0.4303867816925049, "p05": 0.3249378800392151, "p95": 0.5304797887802124 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.9691112041473389, "group": 2, "latest_source_mass": 0.2681158185005188, "layer": 5, "max_source_mass": 0.5506307482719421, "mixer_index": 9, "normalized_entropy": 0.8821230329784768, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.5506307482719421, "median": 0.543993353843689, "p05": 0.41262108087539673, "p95": 0.7161091566085815 }, { "label": "completed_group_1", "mean": 0.18125343322753906, "median": 0.1794438362121582, "p05": 0.11575750261545181, "p95": 0.25031501054763794 }, { "label": "current_group_partial", "mean": 0.2681158185005188, "median": 0.2664111256599426, "p05": 0.1564716249704361, "p95": 0.3817881643772125 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0745112895965576, "group": 2, "latest_source_mass": 0.412794291973114, "layer": 6, "max_source_mass": 0.412794291973114, "mixer_index": 10, "normalized_entropy": 0.9780623252441762, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.34262996912002563, "median": 0.3412924110889435, "p05": 0.29556700587272644, "p95": 0.39125555753707886 }, { "label": "completed_group_1", "mean": 0.24457570910453796, "median": 0.24382859468460083, "p05": 0.22120657563209534, "p95": 0.2734297513961792 }, { "label": "current_group_partial", "mean": 0.412794291973114, "median": 0.4151499271392822, "p05": 0.37735897302627563, "p95": 0.43749260902404785 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0074045658111572, "group": 2, "latest_source_mass": 0.3775783181190491, "layer": 6, "max_source_mass": 0.4399106502532959, "mixer_index": 11, "normalized_entropy": 0.9169791528842925, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.4399106502532959, "median": 0.44856905937194824, "p05": 0.22063353657722473, "p95": 0.6359403133392334 }, { "label": "completed_group_1", "mean": 0.18251106142997742, "median": 0.18562838435173035, "p05": 0.1196163073182106, "p95": 0.23045453429222107 }, { "label": "current_group_partial", "mean": 0.3775783181190491, "median": 0.3675365447998047, "p05": 0.23817461729049683, "p95": 0.5557451844215393 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.060497522354126, "group": 2, "latest_source_mass": 0.4025024175643921, "layer": 7, "max_source_mass": 0.4025024175643921, "mixer_index": 12, "normalized_entropy": 0.9653064445872968, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.3786153197288513, "median": 0.3907395303249359, "p05": 0.28039106726646423, "p95": 0.4436405599117279 }, { "label": "completed_group_1", "mean": 0.2188822478055954, "median": 0.21226292848587036, "p05": 0.1885872334241867, "p95": 0.2672030031681061 }, { "label": "current_group_partial", "mean": 0.4025024175643921, "median": 0.39815032482147217, "p05": 0.3533596992492676, "p95": 0.460100919008255 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 0.9941019415855408, "group": 2, "latest_source_mass": 0.4801652431488037, "layer": 7, "max_source_mass": 0.4801652431488037, "mixer_index": 13, "normalized_entropy": 0.9048705824970601, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.32296955585479736, "median": 0.2803739905357361, "p05": 0.1784922033548355, "p95": 0.6440245509147644 }, { "label": "completed_group_1", "mean": 0.19686520099639893, "median": 0.2053825855255127, "p05": 0.09813719242811203, "p95": 0.2504696547985077 }, { "label": "current_group_partial", "mean": 0.4801652431488037, "median": 0.5057140588760376, "p05": 0.2529003322124481, "p95": 0.598870575428009 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0503522157669067, "group": 2, "latest_source_mass": 0.4001518487930298, "layer": 8, "max_source_mass": 0.40130966901779175, "mixer_index": 14, "normalized_entropy": 0.9560717885654542, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.40130966901779175, "median": 0.396759569644928, "p05": 0.3479611873626709, "p95": 0.4733521044254303 }, { "label": "completed_group_1", "mean": 0.19853851199150085, "median": 0.19769755005836487, "p05": 0.17136667668819427, "p95": 0.22788912057876587 }, { "label": "current_group_partial", "mean": 0.4001518487930298, "median": 0.40291696786880493, "p05": 0.3506356179714203, "p95": 0.43941178917884827 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0129402875900269, "group": 2, "latest_source_mass": 0.42161279916763306, "layer": 8, "max_source_mass": 0.42161279916763306, "mixer_index": 15, "normalized_entropy": 0.9220179839951123, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.4093267023563385, "median": 0.4046899676322937, "p05": 0.2895161807537079, "p95": 0.5762131810188293 }, { "label": "completed_group_1", "mean": 0.16906049847602844, "median": 0.16888955235481262, "p05": 0.11233551055192947, "p95": 0.22477403283119202 }, { "label": "current_group_partial", "mean": 0.42161279916763306, "median": 0.4227708578109741, "p05": 0.3131049573421478, "p95": 0.519075334072113 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0774019956588745, "group": 3, "latest_source_mass": 0.3560434877872467, "layer": 9, "max_source_mass": 0.38510820269584656, "mixer_index": 16, "normalized_entropy": 0.9806935592947451, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.38510820269584656, "median": 0.3892287313938141, "p05": 0.2934730052947998, "p95": 0.4718902111053467 }, { "label": "completed_group_1", "mean": 0.25884830951690674, "median": 0.25734007358551025, "p05": 0.22217319905757904, "p95": 0.29551801085472107 }, { "label": "completed_group_2", "mean": 0.3560434877872467, "median": 0.35412120819091797, "p05": 0.29444968700408936, "p95": 0.41908568143844604 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.2740377187728882, "group": 3, "latest_source_mass": 0.3633861243724823, "layer": 9, "max_source_mass": 0.3633861243724823, "mixer_index": 17, "normalized_entropy": 0.9190239493895668, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.29550763964653015, "median": 0.30360478162765503, "p05": 0.16904757916927338, "p95": 0.45146965980529785 }, { "label": "completed_group_1", "mean": 0.09541244804859161, "median": 0.09486871212720871, "p05": 0.061360105872154236, "p95": 0.13037073612213135 }, { "label": "completed_group_2", "mean": 0.24569380283355713, "median": 0.24333015084266663, "p05": 0.1698218435049057, "p95": 0.31589236855506897 }, { "label": "current_group_partial", "mean": 0.3633861243724823, "median": 0.36472851037979126, "p05": 0.2795379161834717, "p95": 0.4399876892566681 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.360302448272705, "group": 3, "latest_source_mass": 0.28569501638412476, "layer": 10, "max_source_mass": 0.3109527826309204, "mixer_index": 18, "normalized_entropy": 0.9812507981160736, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.3109527826309204, "median": 0.32172977924346924, "p05": 0.23520725965499878, "p95": 0.37052470445632935 }, { "label": "completed_group_1", "mean": 0.20984819531440735, "median": 0.20958667993545532, "p05": 0.1891367882490158, "p95": 0.23184512555599213 }, { "label": "completed_group_2", "mean": 0.19350400567054749, "median": 0.19153150916099548, "p05": 0.16607457399368286, "p95": 0.22877082228660583 }, { "label": "current_group_partial", "mean": 0.28569501638412476, "median": 0.28060320019721985, "p05": 0.23824527859687805, "p95": 0.3439165949821472 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.2722704410552979, "group": 3, "latest_source_mass": 0.29912811517715454, "layer": 10, "max_source_mass": 0.35036465525627136, "mixer_index": 19, "normalized_entropy": 0.9177491279900463, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.35036465525627136, "median": 0.36210548877716064, "p05": 0.12807878851890564, "p95": 0.5797606706619263 }, { "label": "completed_group_1", "mean": 0.12631990015506744, "median": 0.1264108121395111, "p05": 0.06977735459804535, "p95": 0.1755298376083374 }, { "label": "completed_group_2", "mean": 0.22418735921382904, "median": 0.22563719749450684, "p05": 0.1176326721906662, "p95": 0.32001155614852905 }, { "label": "current_group_partial", "mean": 0.29912811517715454, "median": 0.28589946031570435, "p05": 0.193109393119812, "p95": 0.44119665026664734 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3558036088943481, "group": 3, "latest_source_mass": 0.314197838306427, "layer": 11, "max_source_mass": 0.314197838306427, "mixer_index": 20, "normalized_entropy": 0.978005571485618, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.298976868391037, "median": 0.2961939573287964, "p05": 0.24950163066387177, "p95": 0.35835936665534973 }, { "label": "completed_group_1", "mean": 0.1941717267036438, "median": 0.1918032318353653, "p05": 0.16809208691120148, "p95": 0.22895419597625732 }, { "label": "completed_group_2", "mean": 0.1926535815000534, "median": 0.19306811690330505, "p05": 0.16983944177627563, "p95": 0.2155935913324356 }, { "label": "current_group_partial", "mean": 0.314197838306427, "median": 0.31541913747787476, "p05": 0.2673022747039795, "p95": 0.35941022634506226 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.2696614265441895, "group": 3, "latest_source_mass": 0.2862822115421295, "layer": 11, "max_source_mass": 0.3667730689048767, "mixer_index": 21, "normalized_entropy": 0.9158671218416545, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.3667730689048767, "median": 0.4067659378051758, "p05": 0.10593366622924805, "p95": 0.5672274827957153 }, { "label": "completed_group_1", "mean": 0.15310388803482056, "median": 0.15050218999385834, "p05": 0.08651046454906464, "p95": 0.22459912300109863 }, { "label": "completed_group_2", "mean": 0.19384080171585083, "median": 0.18732509016990662, "p05": 0.1267009675502777, "p95": 0.281107634305954 }, { "label": "current_group_partial", "mean": 0.2862822115421295, "median": 0.26271265745162964, "p05": 0.17024393379688263, "p95": 0.475044310092926 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3606936931610107, "group": 3, "latest_source_mass": 0.3028184771537781, "layer": 12, "max_source_mass": 0.3028184771537781, "mixer_index": 22, "normalized_entropy": 0.9815330216461395, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.298227459192276, "median": 0.2954302132129669, "p05": 0.25287362933158875, "p95": 0.35405808687210083 }, { "label": "completed_group_1", "mean": 0.18611131608486176, "median": 0.1850002110004425, "p05": 0.15901757776737213, "p95": 0.21725836396217346 }, { "label": "completed_group_2", "mean": 0.21284276247024536, "median": 0.21402311325073242, "p05": 0.18819917738437653, "p95": 0.23418807983398438 }, { "label": "current_group_partial", "mean": 0.3028184771537781, "median": 0.3041122555732727, "p05": 0.25661808252334595, "p95": 0.34458184242248535 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.269316554069519, "group": 3, "latest_source_mass": 0.2653314769268036, "layer": 12, "max_source_mass": 0.4081852436065674, "mixer_index": 23, "normalized_entropy": 0.9156183489371815, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.4081852436065674, "median": 0.4254342317581177, "p05": 0.20765496790409088, "p95": 0.587238073348999 }, { "label": "completed_group_1", "mean": 0.14420127868652344, "median": 0.1438048928976059, "p05": 0.1064920499920845, "p95": 0.1871720254421234 }, { "label": "completed_group_2", "mean": 0.18228203058242798, "median": 0.17507661879062653, "p05": 0.12183455377817154, "p95": 0.26505547761917114 }, { "label": "current_group_partial", "mean": 0.2653314769268036, "median": 0.24924281239509583, "p05": 0.15609969198703766, "p95": 0.4061311185359955 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3636136054992676, "group": 4, "latest_source_mass": 0.3136237859725952, "layer": 13, "max_source_mass": 0.3136237859725952, "mixer_index": 24, "normalized_entropy": 0.9836392931712563, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.27427661418914795, "median": 0.27403637766838074, "p05": 0.22928935289382935, "p95": 0.319773405790329 }, { "label": "completed_group_1", "mean": 0.21341991424560547, "median": 0.20697325468063354, "p05": 0.18552586436271667, "p95": 0.26688331365585327 }, { "label": "completed_group_2", "mean": 0.19867971539497375, "median": 0.197545126080513, "p05": 0.1794913411140442, "p95": 0.222100168466568 }, { "label": "completed_group_3", "mean": 0.3136237859725952, "median": 0.31875866651535034, "p05": 0.24156072735786438, "p95": 0.3695686459541321 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.570773959159851, "group": 4, "latest_source_mass": 0.2294243425130844, "layer": 13, "max_source_mass": 0.2658740282058716, "mixer_index": 25, "normalized_entropy": 0.9759767351225285, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.2658740282058716, "median": 0.2747163474559784, "p05": 0.17927880585193634, "p95": 0.3512483835220337 }, { "label": "completed_group_1", "mean": 0.13419361412525177, "median": 0.13374894857406616, "p05": 0.11072897166013718, "p95": 0.1586270034313202 }, { "label": "completed_group_2", "mean": 0.1907704919576645, "median": 0.19074805080890656, "p05": 0.14699944853782654, "p95": 0.23316305875778198 }, { "label": "completed_group_3", "mean": 0.17973749339580536, "median": 0.17616114020347595, "p05": 0.1363743096590042, "p95": 0.23202598094940186 }, { "label": "current_group_partial", "mean": 0.2294243425130844, "median": 0.22995705902576447, "p05": 0.17975632846355438, "p95": 0.2789157032966614 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.5560038089752197, "group": 4, "latest_source_mass": 0.32468360662460327, "layer": 14, "max_source_mass": 0.32468360662460327, "mixer_index": 26, "normalized_entropy": 0.9667995248241249, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.20477107167243958, "median": 0.206403911113739, "p05": 0.16619175672531128, "p95": 0.2431035190820694 }, { "label": "completed_group_1", "mean": 0.15721669793128967, "median": 0.15658551454544067, "p05": 0.13878680765628815, "p95": 0.17778633534908295 }, { "label": "completed_group_2", "mean": 0.13811248540878296, "median": 0.13743364810943604, "p05": 0.12352626770734787, "p95": 0.15553249418735504 }, { "label": "completed_group_3", "mean": 0.1752161681652069, "median": 0.17309701442718506, "p05": 0.15223677456378937, "p95": 0.20474471151828766 }, { "label": "current_group_partial", "mean": 0.32468360662460327, "median": 0.32497870922088623, "p05": 0.2946181893348694, "p95": 0.35203880071640015 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.5659973621368408, "group": 4, "latest_source_mass": 0.26703086495399475, "layer": 14, "max_source_mass": 0.26703086495399475, "mixer_index": 27, "normalized_entropy": 0.9730088685238187, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.20531430840492249, "median": 0.20229148864746094, "p05": 0.10698921978473663, "p95": 0.3393132984638214 }, { "label": "completed_group_1", "mean": 0.15986156463623047, "median": 0.15952220559120178, "p05": 0.11969029903411865, "p95": 0.20180393755435944 }, { "label": "completed_group_2", "mean": 0.21557100117206573, "median": 0.21654027700424194, "p05": 0.1673085242509842, "p95": 0.2610440254211426 }, { "label": "completed_group_3", "mean": 0.15222224593162537, "median": 0.1510905921459198, "p05": 0.11800260841846466, "p95": 0.18893754482269287 }, { "label": "current_group_partial", "mean": 0.26703086495399475, "median": 0.258539617061615, "p05": 0.19330690801143646, "p95": 0.36521556973457336 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.5552496910095215, "group": 4, "latest_source_mass": 0.31774693727493286, "layer": 15, "max_source_mass": 0.31774693727493286, "mixer_index": 28, "normalized_entropy": 0.9663309649872576, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.2162160575389862, "median": 0.2145705670118332, "p05": 0.17727568745613098, "p95": 0.26878347992897034 }, { "label": "completed_group_1", "mean": 0.17982497811317444, "median": 0.18024154007434845, "p05": 0.15453973412513733, "p95": 0.2030555009841919 }, { "label": "completed_group_2", "mean": 0.1307673156261444, "median": 0.13107559084892273, "p05": 0.11463287472724915, "p95": 0.14630235731601715 }, { "label": "completed_group_3", "mean": 0.1554446816444397, "median": 0.15516987442970276, "p05": 0.13627474009990692, "p95": 0.1754152774810791 }, { "label": "current_group_partial", "mean": 0.31774693727493286, "median": 0.319039523601532, "p05": 0.27606821060180664, "p95": 0.35650834441185 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.5068919658660889, "group": 4, "latest_source_mass": 0.3298327624797821, "layer": 15, "max_source_mass": 0.3298327624797821, "mixer_index": 29, "normalized_entropy": 0.9362846209998111, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.20646697282791138, "median": 0.18775638937950134, "p05": 0.0806753933429718, "p95": 0.4274584949016571 }, { "label": "completed_group_1", "mean": 0.1197461187839508, "median": 0.11989086866378784, "p05": 0.0809398666024208, "p95": 0.15743419528007507 }, { "label": "completed_group_2", "mean": 0.19215428829193115, "median": 0.19620640575885773, "p05": 0.1148109883069992, "p95": 0.2491212636232376 }, { "label": "completed_group_3", "mean": 0.15179988741874695, "median": 0.15212899446487427, "p05": 0.10697861760854721, "p95": 0.19542087614536285 }, { "label": "current_group_partial", "mean": 0.3298327624797821, "median": 0.32837340235710144, "p05": 0.22746729850769043, "p95": 0.4339301884174347 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.5435223579406738, "group": 4, "latest_source_mass": 0.3224271535873413, "layer": 16, "max_source_mass": 0.3224271535873413, "mixer_index": 30, "normalized_entropy": 0.9590443632623663, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.22410975396633148, "median": 0.22760415077209473, "p05": 0.14593973755836487, "p95": 0.30093055963516235 }, { "label": "completed_group_1", "mean": 0.16986176371574402, "median": 0.16963240504264832, "p05": 0.13569067418575287, "p95": 0.20651353895664215 }, { "label": "completed_group_2", "mean": 0.13080403208732605, "median": 0.13075244426727295, "p05": 0.10945598781108856, "p95": 0.15187400579452515 }, { "label": "completed_group_3", "mean": 0.15279728174209595, "median": 0.15036612749099731, "p05": 0.12810495495796204, "p95": 0.18468116223812103 }, { "label": "current_group_partial", "mean": 0.3224271535873413, "median": 0.3164973258972168, "p05": 0.2600978910923004, "p95": 0.40530598163604736 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.4949920177459717, "group": 4, "latest_source_mass": 0.31847020983695984, "layer": 16, "max_source_mass": 0.31847020983695984, "mixer_index": 31, "normalized_entropy": 0.9288907675133354, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.27174079418182373, "median": 0.2585451304912567, "p05": 0.13639721274375916, "p95": 0.4698348045349121 }, { "label": "completed_group_1", "mean": 0.12178683280944824, "median": 0.11714950203895569, "p05": 0.08594125509262085, "p95": 0.1723237782716751 }, { "label": "completed_group_2", "mean": 0.1529826819896698, "median": 0.15223106741905212, "p05": 0.10371473431587219, "p95": 0.203575000166893 }, { "label": "completed_group_3", "mean": 0.1350194811820984, "median": 0.13363957405090332, "p05": 0.08964549005031586, "p95": 0.18468207120895386 }, { "label": "current_group_partial", "mean": 0.31847020983695984, "median": 0.3185216188430786, "p05": 0.21444711089134216, "p95": 0.4301546812057495 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.544264793395996, "group": 5, "latest_source_mass": 0.2861037850379944, "layer": 17, "max_source_mass": 0.2861037850379944, "mixer_index": 32, "normalized_entropy": 0.9595056643474137, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.24016115069389343, "median": 0.23868846893310547, "p05": 0.18435582518577576, "p95": 0.2983453571796417 }, { "label": "completed_group_1", "mean": 0.20730282366275787, "median": 0.1998077780008316, "p05": 0.16255050897598267, "p95": 0.28107208013534546 }, { "label": "completed_group_2", "mean": 0.09692573547363281, "median": 0.0967404842376709, "p05": 0.08098706603050232, "p95": 0.11221462488174438 }, { "label": "completed_group_3", "mean": 0.1695065200328827, "median": 0.1696585714817047, "p05": 0.136396586894989, "p95": 0.20130634307861328 }, { "label": "completed_group_4", "mean": 0.2861037850379944, "median": 0.28883200883865356, "p05": 0.22401469945907593, "p95": 0.3409370183944702 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.686682939529419, "group": 5, "latest_source_mass": 0.12281867861747742, "layer": 17, "max_source_mass": 0.2885604202747345, "mixer_index": 33, "normalized_entropy": 0.9413556721740635, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.16755425930023193, "median": 0.1327776312828064, "p05": 0.0790192037820816, "p95": 0.3667412996292114 }, { "label": "completed_group_1", "mean": 0.10962115228176117, "median": 0.10525335371494293, "p05": 0.07840172946453094, "p95": 0.15405339002609253 }, { "label": "completed_group_2", "mean": 0.1783454418182373, "median": 0.17896884679794312, "p05": 0.10802385956048965, "p95": 0.24048911035060883 }, { "label": "completed_group_3", "mean": 0.13310004770755768, "median": 0.13033035397529602, "p05": 0.0904831737279892, "p95": 0.17934636771678925 }, { "label": "completed_group_4", "mean": 0.2885604202747345, "median": 0.2831825315952301, "p05": 0.16967125236988068, "p95": 0.4189499616622925 }, { "label": "current_group_partial", "mean": 0.12281867861747742, "median": 0.12062528729438782, "p05": 0.07461707293987274, "p95": 0.1782219111919403 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.7070834636688232, "group": 5, "latest_source_mass": 0.2988377809524536, "layer": 18, "max_source_mass": 0.2988377809524536, "mixer_index": 34, "normalized_entropy": 0.9527414214834803, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.181177020072937, "median": 0.18419909477233887, "p05": 0.13624237477779388, "p95": 0.22192169725894928 }, { "label": "completed_group_1", "mean": 0.14557164907455444, "median": 0.144476979970932, "p05": 0.1293315887451172, "p95": 0.16486899554729462 }, { "label": "completed_group_2", "mean": 0.08318156749010086, "median": 0.0820261612534523, "p05": 0.07088497281074524, "p95": 0.0990293100476265 }, { "label": "completed_group_3", "mean": 0.11424209177494049, "median": 0.11267061531543732, "p05": 0.0979941189289093, "p95": 0.1353992223739624 }, { "label": "completed_group_4", "mean": 0.17698991298675537, "median": 0.17597860097885132, "p05": 0.14837387204170227, "p95": 0.21050144731998444 }, { "label": "current_group_partial", "mean": 0.2988377809524536, "median": 0.2989804744720459, "p05": 0.2721370458602905, "p95": 0.32572072744369507 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.7419235706329346, "group": 5, "latest_source_mass": 0.26016175746917725, "layer": 18, "max_source_mass": 0.26016175746917725, "mixer_index": 35, "normalized_entropy": 0.972186055410333, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1532188355922699, "median": 0.1451343297958374, "p05": 0.11514341086149216, "p95": 0.22103679180145264 }, { "label": "completed_group_1", "mean": 0.11420352011919022, "median": 0.11261284351348877, "p05": 0.0928458571434021, "p95": 0.14123988151550293 }, { "label": "completed_group_2", "mean": 0.16597424447536469, "median": 0.1655328869819641, "p05": 0.13871800899505615, "p95": 0.19611573219299316 }, { "label": "completed_group_3", "mean": 0.1250365972518921, "median": 0.12369821965694427, "p05": 0.10375828295946121, "p95": 0.14912742376327515 }, { "label": "completed_group_4", "mean": 0.18140508234500885, "median": 0.1796584278345108, "p05": 0.14511024951934814, "p95": 0.22286373376846313 }, { "label": "current_group_partial", "mean": 0.26016175746917725, "median": 0.26266711950302124, "p05": 0.1889389455318451, "p95": 0.3234381675720215 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.7127439975738525, "group": 5, "latest_source_mass": 0.2907913327217102, "layer": 19, "max_source_mass": 0.2907913327217102, "mixer_index": 36, "normalized_entropy": 0.9559006256078307, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.18731768429279327, "median": 0.19042012095451355, "p05": 0.13879702985286713, "p95": 0.23792000114917755 }, { "label": "completed_group_1", "mean": 0.16715484857559204, "median": 0.1667211651802063, "p05": 0.14574480056762695, "p95": 0.19026139378547668 }, { "label": "completed_group_2", "mean": 0.08496320247650146, "median": 0.08452782034873962, "p05": 0.07410907745361328, "p95": 0.097386933863163 }, { "label": "completed_group_3", "mean": 0.11362075805664062, "median": 0.11248823255300522, "p05": 0.0977015346288681, "p95": 0.13219426572322845 }, { "label": "completed_group_4", "mean": 0.15615218877792358, "median": 0.15540224313735962, "p05": 0.1289147436618805, "p95": 0.18598109483718872 }, { "label": "current_group_partial", "mean": 0.2907913327217102, "median": 0.2900415062904358, "p05": 0.25822532176971436, "p95": 0.32475030422210693 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.6076834201812744, "group": 5, "latest_source_mass": 0.3738824725151062, "layer": 19, "max_source_mass": 0.3738824725151062, "mixer_index": 37, "normalized_entropy": 0.8972652009334232, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.15032559633255005, "median": 0.12544110417366028, "p05": 0.06976518034934998, "p95": 0.35840943455696106 }, { "label": "completed_group_1", "mean": 0.08624927699565887, "median": 0.08602266013622284, "p05": 0.061998721212148666, "p95": 0.11138717085123062 }, { "label": "completed_group_2", "mean": 0.15736976265907288, "median": 0.15802210569381714, "p05": 0.10121746361255646, "p95": 0.20687000453472137 }, { "label": "completed_group_3", "mean": 0.08584291487932205, "median": 0.0827617198228836, "p05": 0.060561805963516235, "p95": 0.11351694166660309 }, { "label": "completed_group_4", "mean": 0.14632999897003174, "median": 0.1426895558834076, "p05": 0.10106660425662994, "p95": 0.20179018378257751 }, { "label": "current_group_partial", "mean": 0.3738824725151062, "median": 0.3799295425415039, "p05": 0.23631010949611664, "p95": 0.4884527325630188 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.709290862083435, "group": 5, "latest_source_mass": 0.2901708781719208, "layer": 20, "max_source_mass": 0.2901708781719208, "mixer_index": 38, "normalized_entropy": 0.9539733939957076, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.18991519510746002, "median": 0.1924753338098526, "p05": 0.14201581478118896, "p95": 0.23709386587142944 }, { "label": "completed_group_1", "mean": 0.17091524600982666, "median": 0.17006167769432068, "p05": 0.15297070145606995, "p95": 0.19153237342834473 }, { "label": "completed_group_2", "mean": 0.07822999358177185, "median": 0.07778562605381012, "p05": 0.06707640737295151, "p95": 0.09089364111423492 }, { "label": "completed_group_3", "mean": 0.116765096783638, "median": 0.11516137421131134, "p05": 0.10162825882434845, "p95": 0.13653048872947693 }, { "label": "completed_group_4", "mean": 0.1540035903453827, "median": 0.15287795662879944, "p05": 0.12593276798725128, "p95": 0.18590238690376282 }, { "label": "current_group_partial", "mean": 0.2901708781719208, "median": 0.29104921221733093, "p05": 0.25643545389175415, "p95": 0.3205650746822357 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.6690887212753296, "group": 5, "latest_source_mass": 0.3342471718788147, "layer": 20, "max_source_mass": 0.3342471718788147, "mixer_index": 39, "normalized_entropy": 0.9315361520005944, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.19029226899147034, "median": 0.19644704461097717, "p05": 0.09456793963909149, "p95": 0.30468082427978516 }, { "label": "completed_group_1", "mean": 0.1100241169333458, "median": 0.10808776319026947, "p05": 0.09035106748342514, "p95": 0.1359245330095291 }, { "label": "completed_group_2", "mean": 0.11471512168645859, "median": 0.11494918912649155, "p05": 0.08799737691879272, "p95": 0.14024731516838074 }, { "label": "completed_group_3", "mean": 0.09999909996986389, "median": 0.09682698547840118, "p05": 0.07921088486909866, "p95": 0.1264391839504242 }, { "label": "completed_group_4", "mean": 0.1507222205400467, "median": 0.1476232260465622, "p05": 0.11397356539964676, "p95": 0.19991227984428406 }, { "label": "current_group_partial", "mean": 0.3342471718788147, "median": 0.33802270889282227, "p05": 0.2548266053199768, "p95": 0.40130624175071716 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.7458693981170654, "group": 6, "latest_source_mass": 0.2361394464969635, "layer": 21, "max_source_mass": 0.2361394464969635, "mixer_index": 40, "normalized_entropy": 0.9743882636597644, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.20528709888458252, "median": 0.2140858769416809, "p05": 0.13168928027153015, "p95": 0.27098023891448975 }, { "label": "completed_group_1", "mean": 0.1722964197397232, "median": 0.17238952219486237, "p05": 0.15053534507751465, "p95": 0.19365043938159943 }, { "label": "completed_group_2", "mean": 0.10435444861650467, "median": 0.10361101478338242, "p05": 0.08888213336467743, "p95": 0.12212536484003067 }, { "label": "completed_group_3", "mean": 0.12075909972190857, "median": 0.1180175170302391, "p05": 0.10358668118715286, "p95": 0.14458929002285004 }, { "label": "completed_group_4", "mean": 0.16116347908973694, "median": 0.15841388702392578, "p05": 0.1337081342935562, "p95": 0.19791515171527863 }, { "label": "completed_group_5", "mean": 0.2361394464969635, "median": 0.23709020018577576, "p05": 0.20446214079856873, "p95": 0.26509594917297363 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.4612125158309937, "group": 6, "latest_source_mass": 0.5543455481529236, "layer": 21, "max_source_mass": 0.5543455481529236, "mixer_index": 41, "normalized_entropy": 0.7509146897354808, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.08934466540813446, "median": 0.07911523431539536, "p05": 0.04488275945186615, "p95": 0.1737232506275177 }, { "label": "completed_group_1", "mean": 0.054379627108573914, "median": 0.05344422906637192, "p05": 0.04221750423312187, "p95": 0.06924191862344742 }, { "label": "completed_group_2", "mean": 0.07567165046930313, "median": 0.07438549399375916, "p05": 0.05811355262994766, "p95": 0.09632256627082825 }, { "label": "completed_group_3", "mean": 0.05758609622716904, "median": 0.05612001568078995, "p05": 0.044792283326387405, "p95": 0.07347827404737473 }, { "label": "completed_group_4", "mean": 0.07884950935840607, "median": 0.07757881283760071, "p05": 0.05758974328637123, "p95": 0.10452556610107422 }, { "label": "completed_group_5", "mean": 0.08982288837432861, "median": 0.08912864327430725, "p05": 0.0663854256272316, "p95": 0.11604885756969452 }, { "label": "current_group_partial", "mean": 0.5543455481529236, "median": 0.5596089363098145, "p05": 0.45455610752105713, "p95": 0.6367484331130981 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.7241814136505127, "group": 6, "latest_source_mass": 0.39484071731567383, "layer": 22, "max_source_mass": 0.39484071731567383, "mixer_index": 42, "normalized_entropy": 0.886053970419732, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.13360010087490082, "median": 0.14072942733764648, "p05": 0.07699814438819885, "p95": 0.17506597936153412 }, { "label": "completed_group_1", "mean": 0.10609637200832367, "median": 0.10586297512054443, "p05": 0.08648421615362167, "p95": 0.12602579593658447 }, { "label": "completed_group_2", "mean": 0.06363911926746368, "median": 0.06324908882379532, "p05": 0.054980404675006866, "p95": 0.07316673547029495 }, { "label": "completed_group_3", "mean": 0.07426095008850098, "median": 0.07362888753414154, "p05": 0.06318790465593338, "p95": 0.08665015548467636 }, { "label": "completed_group_4", "mean": 0.09383076429367065, "median": 0.09294568747282028, "p05": 0.07638302445411682, "p95": 0.11445100605487823 }, { "label": "completed_group_5", "mean": 0.13373197615146637, "median": 0.13327741622924805, "p05": 0.11399447172880173, "p95": 0.15641780197620392 }, { "label": "current_group_partial", "mean": 0.39484071731567383, "median": 0.3879151940345764, "p05": 0.3302172124385834, "p95": 0.4852263331413269 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.3979300260543823, "group": 6, "latest_source_mass": 0.5830056071281433, "layer": 22, "max_source_mass": 0.5830056071281433, "mixer_index": 43, "normalized_entropy": 0.7183939231382495, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.08959861099720001, "median": 0.07544538378715515, "p05": 0.04165133833885193, "p95": 0.20099139213562012 }, { "label": "completed_group_1", "mean": 0.05414731800556183, "median": 0.05336771905422211, "p05": 0.04069708660244942, "p95": 0.07046990841627121 }, { "label": "completed_group_2", "mean": 0.07668881118297577, "median": 0.07607124000787735, "p05": 0.0555989071726799, "p95": 0.09980960935354233 }, { "label": "completed_group_3", "mean": 0.05603286623954773, "median": 0.054573021829128265, "p05": 0.04162490367889404, "p95": 0.0730118378996849 }, { "label": "completed_group_4", "mean": 0.06979228556156158, "median": 0.06884932518005371, "p05": 0.04576495662331581, "p95": 0.09806846082210541 }, { "label": "completed_group_5", "mean": 0.07073447108268738, "median": 0.07023533433675766, "p05": 0.04956750199198723, "p95": 0.09379896521568298 }, { "label": "current_group_partial", "mean": 0.5830056071281433, "median": 0.5862616896629333, "p05": 0.4658474922180176, "p95": 0.6849814057350159 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.7459560632705688, "group": 6, "latest_source_mass": 0.37691548466682434, "layer": 23, "max_source_mass": 0.37691548466682434, "mixer_index": 44, "normalized_entropy": 0.897243926765161, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.14318783581256866, "median": 0.1493408977985382, "p05": 0.08292607218027115, "p95": 0.19464610517024994 }, { "label": "completed_group_1", "mean": 0.1094653457403183, "median": 0.10858799517154694, "p05": 0.09432064741849899, "p95": 0.1279931515455246 }, { "label": "completed_group_2", "mean": 0.06349228322505951, "median": 0.06325754523277283, "p05": 0.05439673364162445, "p95": 0.0737837627530098 }, { "label": "completed_group_3", "mean": 0.07859592139720917, "median": 0.0775163471698761, "p05": 0.06603655964136124, "p95": 0.09357087314128876 }, { "label": "completed_group_4", "mean": 0.09567482024431229, "median": 0.09468592703342438, "p05": 0.07671141624450684, "p95": 0.11774884164333344 }, { "label": "completed_group_5", "mean": 0.13266831636428833, "median": 0.13199329376220703, "p05": 0.11244359612464905, "p95": 0.15504513680934906 }, { "label": "current_group_partial", "mean": 0.37691548466682434, "median": 0.37180638313293457, "p05": 0.31207239627838135, "p95": 0.4595233201980591 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.4055393934249878, "group": 6, "latest_source_mass": 0.5772827863693237, "layer": 23, "max_source_mass": 0.5772827863693237, "mixer_index": 45, "normalized_entropy": 0.7223043644164862, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.10154955089092255, "median": 0.08766824007034302, "p05": 0.04901552200317383, "p95": 0.20287738740444183 }, { "label": "completed_group_1", "mean": 0.05619276314973831, "median": 0.055477917194366455, "p05": 0.041608311235904694, "p95": 0.07371112704277039 }, { "label": "completed_group_2", "mean": 0.07791627198457718, "median": 0.07736755907535553, "p05": 0.055920060724020004, "p95": 0.10158214718103409 }, { "label": "completed_group_3", "mean": 0.053915880620479584, "median": 0.05215863138437271, "p05": 0.03798118978738785, "p95": 0.07156119495630264 }, { "label": "completed_group_4", "mean": 0.06398355215787888, "median": 0.06229078769683838, "p05": 0.040616974234580994, "p95": 0.09270931780338287 }, { "label": "completed_group_5", "mean": 0.06915920227766037, "median": 0.0684593990445137, "p05": 0.046986356377601624, "p95": 0.09408729523420334 }, { "label": "current_group_partial", "mean": 0.5772827863693237, "median": 0.5824689269065857, "p05": 0.4518115520477295, "p95": 0.6926202178001404 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.7853620052337646, "group": 6, "latest_source_mass": 0.3407441973686218, "layer": 24, "max_source_mass": 0.3407441973686218, "mixer_index": 46, "normalized_entropy": 0.9174945750195659, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.15284113585948944, "median": 0.1593906283378601, "p05": 0.08678612112998962, "p95": 0.211342453956604 }, { "label": "completed_group_1", "mean": 0.12206703424453735, "median": 0.12153112143278122, "p05": 0.1060115247964859, "p95": 0.1399766057729721 }, { "label": "completed_group_2", "mean": 0.06508444249629974, "median": 0.06482555717229843, "p05": 0.055350616574287415, "p95": 0.07557765394449234 }, { "label": "completed_group_3", "mean": 0.08347313106060028, "median": 0.08190013468265533, "p05": 0.06966532766819, "p95": 0.10050469636917114 }, { "label": "completed_group_4", "mean": 0.09811970591545105, "median": 0.09664031863212585, "p05": 0.07688035070896149, "p95": 0.12369741499423981 }, { "label": "completed_group_5", "mean": 0.13767032325267792, "median": 0.1373325139284134, "p05": 0.11642646044492722, "p95": 0.1606694608926773 }, { "label": "current_group_partial", "mean": 0.3407441973686218, "median": 0.33805108070373535, "p05": 0.271301805973053, "p95": 0.41781002283096313 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.509842872619629, "group": 6, "latest_source_mass": 0.5227814316749573, "layer": 24, "max_source_mass": 0.5227814316749573, "mixer_index": 47, "normalized_entropy": 0.7759057494780099, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.11788781732320786, "median": 0.10097194463014603, "p05": 0.06265684217214584, "p95": 0.2408325970172882 }, { "label": "completed_group_1", "mean": 0.06822571158409119, "median": 0.0673636943101883, "p05": 0.05209728330373764, "p95": 0.08648915588855743 }, { "label": "completed_group_2", "mean": 0.08556114137172699, "median": 0.08463940769433975, "p05": 0.06234566867351532, "p95": 0.1125786080956459 }, { "label": "completed_group_3", "mean": 0.06067090481519699, "median": 0.058974385261535645, "p05": 0.043267179280519485, "p95": 0.07997901737689972 }, { "label": "completed_group_4", "mean": 0.07228855788707733, "median": 0.06972275674343109, "p05": 0.04689513146877289, "p95": 0.106512151658535 }, { "label": "completed_group_5", "mean": 0.07258446514606476, "median": 0.07082539796829224, "p05": 0.050949547439813614, "p95": 0.09970614314079285 }, { "label": "current_group_partial", "mean": 0.5227814316749573, "median": 0.5290273427963257, "p05": 0.38773614168167114, "p95": 0.6392589807510376 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.839367389678955, "group": 7, "latest_source_mass": 0.29583215713500977, "layer": 25, "max_source_mass": 0.29583215713500977, "mixer_index": 48, "normalized_entropy": 0.9452478525649903, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.15906387567520142, "median": 0.15846745669841766, "p05": 0.09519976377487183, "p95": 0.24517810344696045 }, { "label": "completed_group_1", "mean": 0.13139380514621735, "median": 0.1308426856994629, "p05": 0.11478274315595627, "p95": 0.15000829100608826 }, { "label": "completed_group_2", "mean": 0.07717745751142502, "median": 0.07715500891208649, "p05": 0.06616044044494629, "p95": 0.08791667222976685 }, { "label": "completed_group_3", "mean": 0.09607303142547607, "median": 0.09436552226543427, "p05": 0.08152313530445099, "p95": 0.11384639143943787 }, { "label": "completed_group_4", "mean": 0.1107850894331932, "median": 0.1089654266834259, "p05": 0.08814431726932526, "p95": 0.139452263712883 }, { "label": "completed_group_5", "mean": 0.12967455387115479, "median": 0.1299237608909607, "p05": 0.10832332074642181, "p95": 0.15138982236385345 }, { "label": "completed_group_6", "mean": 0.29583215713500977, "median": 0.29647016525268555, "p05": 0.2276897132396698, "p95": 0.3604946434497833 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.5841834545135498, "group": 7, "latest_source_mass": 0.40767741203308105, "layer": 25, "max_source_mass": 0.40767741203308105, "mixer_index": 49, "normalized_entropy": 0.7618312045616817, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.07896879315376282, "median": 0.06809917092323303, "p05": 0.025932887569069862, "p95": 0.19368338584899902 }, { "label": "completed_group_1", "mean": 0.048752427101135254, "median": 0.04767637699842453, "p05": 0.034443892538547516, "p95": 0.0667153000831604 }, { "label": "completed_group_2", "mean": 0.055707164108753204, "median": 0.0549345538020134, "p05": 0.038696639239788055, "p95": 0.0750277042388916 }, { "label": "completed_group_3", "mean": 0.04139826446771622, "median": 0.03878612071275711, "p05": 0.026673054322600365, "p95": 0.05746251344680786 }, { "label": "completed_group_4", "mean": 0.04449018836021423, "median": 0.04196460545063019, "p05": 0.026162121444940567, "p95": 0.07164917886257172 }, { "label": "completed_group_5", "mean": 0.04285799711942673, "median": 0.04109475016593933, "p05": 0.029393533244729042, "p95": 0.06277710199356079 }, { "label": "completed_group_6", "mean": 0.2801477909088135, "median": 0.2763082981109619, "p05": 0.1455708146095276, "p95": 0.4254714846611023 }, { "label": "current_group_partial", "mean": 0.40767741203308105, "median": 0.4120897948741913, "p05": 0.21399420499801636, "p95": 0.5853031277656555 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.8125314712524414, "group": 7, "latest_source_mass": 0.3700707256793976, "layer": 26, "max_source_mass": 0.3700707256793976, "mixer_index": 50, "normalized_entropy": 0.8716433883436914, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.10889513790607452, "median": 0.11333495378494263, "p05": 0.04850132763385773, "p95": 0.1620434820652008 }, { "label": "completed_group_1", "mean": 0.08656036853790283, "median": 0.08640051633119583, "p05": 0.0650845319032669, "p95": 0.1084962785243988 }, { "label": "completed_group_2", "mean": 0.048123255372047424, "median": 0.04800213500857353, "p05": 0.03576180338859558, "p95": 0.06107594817876816 }, { "label": "completed_group_3", "mean": 0.06561170518398285, "median": 0.06379580497741699, "p05": 0.050968848168849945, "p95": 0.08332888036966324 }, { "label": "completed_group_4", "mean": 0.06917259097099304, "median": 0.06724178791046143, "p05": 0.050207968801259995, "p95": 0.09373342245817184 }, { "label": "completed_group_5", "mean": 0.08286689221858978, "median": 0.08183546364307404, "p05": 0.06535022705793381, "p95": 0.10340933501720428 }, { "label": "completed_group_6", "mean": 0.16869932413101196, "median": 0.16509854793548584, "p05": 0.11059704422950745, "p95": 0.23654375970363617 }, { "label": "current_group_partial", "mean": 0.3700707256793976, "median": 0.3694133758544922, "p05": 0.23441238701343536, "p95": 0.5025027990341187 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.50089693069458, "group": 7, "latest_source_mass": 0.49096518754959106, "layer": 26, "max_source_mass": 0.49096518754959106, "mixer_index": 51, "normalized_entropy": 0.7217788529328457, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.07974998652935028, "median": 0.07402504235506058, "p05": 0.04579866677522659, "p95": 0.1359037309885025 }, { "label": "completed_group_1", "mean": 0.04166398197412491, "median": 0.04038450866937637, "p05": 0.027992289513349533, "p95": 0.05969971418380737 }, { "label": "completed_group_2", "mean": 0.04607272148132324, "median": 0.044438350945711136, "p05": 0.030038591474294662, "p95": 0.06808309257030487 }, { "label": "completed_group_3", "mean": 0.03586764633655548, "median": 0.03317485377192497, "p05": 0.02203454077243805, "p95": 0.05572832003235817 }, { "label": "completed_group_4", "mean": 0.04223340004682541, "median": 0.03930673003196716, "p05": 0.02465122565627098, "p95": 0.06968094408512115 }, { "label": "completed_group_5", "mean": 0.04095568507909775, "median": 0.03919431194663048, "p05": 0.026239857077598572, "p95": 0.06058088690042496 }, { "label": "completed_group_6", "mean": 0.22249138355255127, "median": 0.2125401496887207, "p05": 0.10647299885749817, "p95": 0.3711881935596466 }, { "label": "current_group_partial", "mean": 0.49096518754959106, "median": 0.4988659620285034, "p05": 0.2878011167049408, "p95": 0.6676807403564453 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.8059929609298706, "group": 7, "latest_source_mass": 0.37139976024627686, "layer": 27, "max_source_mass": 0.37139976024627686, "mixer_index": 52, "normalized_entropy": 0.8684990295379666, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.10711923241615295, "median": 0.10972198843955994, "p05": 0.04896187782287598, "p95": 0.16201043128967285 }, { "label": "completed_group_1", "mean": 0.07651707530021667, "median": 0.07666032761335373, "p05": 0.05141128972172737, "p95": 0.10152198374271393 }, { "label": "completed_group_2", "mean": 0.058284737169742584, "median": 0.05812416970729828, "p05": 0.039273619651794434, "p95": 0.07745087146759033 }, { "label": "completed_group_3", "mean": 0.06609587371349335, "median": 0.06383530795574188, "p05": 0.04481930285692215, "p95": 0.09336265176534653 }, { "label": "completed_group_4", "mean": 0.07139429450035095, "median": 0.0694514587521553, "p05": 0.0485839769244194, "p95": 0.10034126043319702 }, { "label": "completed_group_5", "mean": 0.07493151724338531, "median": 0.07319007813930511, "p05": 0.051876723766326904, "p95": 0.10402432829141617 }, { "label": "completed_group_6", "mean": 0.17425751686096191, "median": 0.16937679052352905, "p05": 0.10449051856994629, "p95": 0.2624811828136444 }, { "label": "current_group_partial", "mean": 0.37139976024627686, "median": 0.3732905089855194, "p05": 0.23830299079418182, "p95": 0.5014542937278748 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.5622732639312744, "group": 7, "latest_source_mass": 0.4768889844417572, "layer": 27, "max_source_mass": 0.4768889844417572, "mixer_index": 53, "normalized_entropy": 0.7512946301290215, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.08775827288627625, "median": 0.08504067361354828, "p05": 0.0428270623087883, "p95": 0.144285649061203 }, { "label": "completed_group_1", "mean": 0.047765426337718964, "median": 0.04646281152963638, "p05": 0.031814731657505035, "p95": 0.06807165592908859 }, { "label": "completed_group_2", "mean": 0.053928617388010025, "median": 0.052262187004089355, "p05": 0.035437777638435364, "p95": 0.0784272700548172 }, { "label": "completed_group_3", "mean": 0.04293380305171013, "median": 0.03996016085147858, "p05": 0.02556057646870613, "p95": 0.06623870134353638 }, { "label": "completed_group_4", "mean": 0.04174989461898804, "median": 0.03903091698884964, "p05": 0.025013400241732597, "p95": 0.06869364529848099 }, { "label": "completed_group_5", "mean": 0.04734005779027939, "median": 0.04609068110585213, "p05": 0.029884062707424164, "p95": 0.06911227852106094 }, { "label": "completed_group_6", "mean": 0.20163494348526, "median": 0.19222964346408844, "p05": 0.09963390231132507, "p95": 0.3307305574417114 }, { "label": "current_group_partial", "mean": 0.4768889844417572, "median": 0.48383551836013794, "p05": 0.276212215423584, "p95": 0.6487575173377991 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.8757455348968506, "group": 7, "latest_source_mass": 0.3109493851661682, "layer": 28, "max_source_mass": 0.3109493851661682, "mixer_index": 54, "normalized_entropy": 0.9020429270551009, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.11563669890165329, "median": 0.12087491154670715, "p05": 0.05733206868171692, "p95": 0.1688953936100006 }, { "label": "completed_group_1", "mean": 0.08108697831630707, "median": 0.08185897022485733, "p05": 0.05175650864839554, "p95": 0.10753051191568375 }, { "label": "completed_group_2", "mean": 0.07280858606100082, "median": 0.07357072830200195, "p05": 0.04870978742837906, "p95": 0.0948689877986908 }, { "label": "completed_group_3", "mean": 0.07553227245807648, "median": 0.07362619787454605, "p05": 0.056317005306482315, "p95": 0.09756095707416534 }, { "label": "completed_group_4", "mean": 0.07606014609336853, "median": 0.07456053793430328, "p05": 0.05528895929455757, "p95": 0.1022825762629509 }, { "label": "completed_group_5", "mean": 0.06935927271842957, "median": 0.06843917071819305, "p05": 0.04978285729885101, "p95": 0.09167985618114471 }, { "label": "completed_group_6", "mean": 0.19856664538383484, "median": 0.19508783519268036, "p05": 0.1281973123550415, "p95": 0.27796900272369385 }, { "label": "current_group_partial", "mean": 0.3109493851661682, "median": 0.3105033040046692, "p05": 0.18172834813594818, "p95": 0.43916669487953186 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.6924852132797241, "group": 7, "latest_source_mass": 0.3315581679344177, "layer": 28, "max_source_mass": 0.3315581679344177, "mixer_index": 55, "normalized_entropy": 0.8139133413255193, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.08638577908277512, "median": 0.058608733117580414, "p05": 0.026544082909822464, "p95": 0.2740764319896698 }, { "label": "completed_group_1", "mean": 0.059912845492362976, "median": 0.05901291221380234, "p05": 0.03629481792449951, "p95": 0.08707985281944275 }, { "label": "completed_group_2", "mean": 0.06609142571687698, "median": 0.0647939145565033, "p05": 0.038691695779561996, "p95": 0.09703366458415985 }, { "label": "completed_group_3", "mean": 0.056745611131191254, "median": 0.05418796092271805, "p05": 0.032174739986658096, "p95": 0.08370660990476608 }, { "label": "completed_group_4", "mean": 0.06277773529291153, "median": 0.0600711889564991, "p05": 0.03501531109213829, "p95": 0.10094858705997467 }, { "label": "completed_group_5", "mean": 0.05352145433425903, "median": 0.05179818347096443, "p05": 0.032271772623062134, "p95": 0.08084158599376678 }, { "label": "completed_group_6", "mean": 0.2830069661140442, "median": 0.2759752869606018, "p05": 0.12981083989143372, "p95": 0.46204638481140137 }, { "label": "current_group_partial", "mean": 0.3315581679344177, "median": 0.3204721510410309, "p05": 0.13446375727653503, "p95": 0.5667697787284851 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.8980522155761719, "group": 8, "latest_source_mass": 0.20123262703418732, "layer": 29, "max_source_mass": 0.26602888107299805, "mixer_index": 56, "normalized_entropy": 0.9127701729200177, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.11603894829750061, "median": 0.09991352260112762, "p05": 0.030035395175218582, "p95": 0.2941743731498718 }, { "label": "completed_group_1", "mean": 0.08804173022508621, "median": 0.08880250155925751, "p05": 0.05664174258708954, "p95": 0.11779069900512695 }, { "label": "completed_group_2", "mean": 0.07975024729967117, "median": 0.07984748482704163, "p05": 0.05428915470838547, "p95": 0.10499341040849686 }, { "label": "completed_group_3", "mean": 0.09027000516653061, "median": 0.08898856490850449, "p05": 0.06189651042222977, "p95": 0.11996553838253021 }, { "label": "completed_group_4", "mean": 0.08430293202400208, "median": 0.0819794088602066, "p05": 0.05464346706867218, "p95": 0.12159879505634308 }, { "label": "completed_group_5", "mean": 0.07433462142944336, "median": 0.07287181913852692, "p05": 0.052520088851451874, "p95": 0.1010315865278244 }, { "label": "completed_group_6", "mean": 0.26602888107299805, "median": 0.2639757990837097, "p05": 0.15474572777748108, "p95": 0.3846370279788971 }, { "label": "completed_group_7", "mean": 0.20123262703418732, "median": 0.18140269815921783, "p05": 0.08419936150312424, "p95": 0.3782624900341034 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.7512003183364868, "group": 8, "latest_source_mass": 0.33482813835144043, "layer": 29, "max_source_mass": 0.33482813835144043, "mixer_index": 57, "normalized_entropy": 0.7970056117156376, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.07744461297988892, "median": 0.06970229744911194, "p05": 0.014401955530047417, "p95": 0.17144526541233063 }, { "label": "completed_group_1", "mean": 0.046086348593235016, "median": 0.04394831880927086, "p05": 0.028066720813512802, "p95": 0.0718703418970108 }, { "label": "completed_group_2", "mean": 0.05025433003902435, "median": 0.04861236736178398, "p05": 0.029644077643752098, "p95": 0.07761462777853012 }, { "label": "completed_group_3", "mean": 0.03748488053679466, "median": 0.03434944897890091, "p05": 0.020774798467755318, "p95": 0.05938909202814102 }, { "label": "completed_group_4", "mean": 0.04005061462521553, "median": 0.037498839199543, "p05": 0.019541850313544273, "p95": 0.06922487914562225 }, { "label": "completed_group_5", "mean": 0.041623204946517944, "median": 0.039552345871925354, "p05": 0.022081367671489716, "p95": 0.06821337342262268 }, { "label": "completed_group_6", "mean": 0.198423832654953, "median": 0.18318486213684082, "p05": 0.07548617571592331, "p95": 0.3721975088119507 }, { "label": "completed_group_7", "mean": 0.17380405962467194, "median": 0.1514919549226761, "p05": 0.05777408182621002, "p95": 0.36326974630355835 }, { "label": "current_group_partial", "mean": 0.33482813835144043, "median": 0.33602219820022583, "p05": 0.0795319601893425, "p95": 0.5979493856430054 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.0160694122314453, "group": 8, "latest_source_mass": 0.21711516380310059, "layer": 30, "max_source_mass": 0.21711516380310059, "mixer_index": 58, "normalized_entropy": 0.9175527313077866, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.09385271370410919, "median": 0.09211117774248123, "p05": 0.032377734780311584, "p95": 0.16496515274047852 }, { "label": "completed_group_1", "mean": 0.06208505108952522, "median": 0.06140740588307381, "p05": 0.0407925546169281, "p95": 0.0863042101264 }, { "label": "completed_group_2", "mean": 0.06560768187046051, "median": 0.06516212224960327, "p05": 0.041897669434547424, "p95": 0.09085705131292343 }, { "label": "completed_group_3", "mean": 0.07029052078723907, "median": 0.06780634075403214, "p05": 0.04846132919192314, "p95": 0.09917552024126053 }, { "label": "completed_group_4", "mean": 0.06935097277164459, "median": 0.06708167493343353, "p05": 0.04668639227747917, "p95": 0.09815650433301926 }, { "label": "completed_group_5", "mean": 0.056832700967788696, "median": 0.05595892667770386, "p05": 0.038822539150714874, "p95": 0.0771557092666626 }, { "label": "completed_group_6", "mean": 0.19425815343856812, "median": 0.19145764410495758, "p05": 0.12249615788459778, "p95": 0.27496030926704407 }, { "label": "completed_group_7", "mean": 0.17060703039169312, "median": 0.16317546367645264, "p05": 0.09239593893289566, "p95": 0.2726203501224518 }, { "label": "current_group_partial", "mean": 0.21711516380310059, "median": 0.21198657155036926, "p05": 0.11019522696733475, "p95": 0.3391069769859314 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 1.7837426662445068, "group": 8, "latest_source_mass": 0.29026859998703003, "layer": 30, "max_source_mass": 0.29026859998703003, "mixer_index": 59, "normalized_entropy": 0.8118162725118464, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.08836327493190765, "median": 0.07904411852359772, "p05": 0.02631576545536518, "p95": 0.17534467577934265 }, { "label": "completed_group_1", "mean": 0.04953558370471001, "median": 0.04658038914203644, "p05": 0.025181813165545464, "p95": 0.08318150043487549 }, { "label": "completed_group_2", "mean": 0.04813014343380928, "median": 0.045740917325019836, "p05": 0.026714935898780823, "p95": 0.07843323051929474 }, { "label": "completed_group_3", "mean": 0.0373263955116272, "median": 0.03401809185743332, "p05": 0.01990434154868126, "p95": 0.06290259957313538 }, { "label": "completed_group_4", "mean": 0.037015400826931, "median": 0.03405017405748367, "p05": 0.018646355718374252, "p95": 0.06544196605682373 }, { "label": "completed_group_5", "mean": 0.03390160948038101, "median": 0.03198609501123428, "p05": 0.018189728260040283, "p95": 0.05570102855563164 }, { "label": "completed_group_6", "mean": 0.1840541511774063, "median": 0.17327937483787537, "p05": 0.07966600358486176, "p95": 0.32982927560806274 }, { "label": "completed_group_7", "mean": 0.23140482604503632, "median": 0.22406929731369019, "p05": 0.08831030130386353, "p95": 0.40350067615509033 }, { "label": "current_group_partial", "mean": 0.29026859998703003, "median": 0.2745037376880646, "p05": 0.10280418395996094, "p95": 0.5399085879325867 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 1.9973840713500977, "group": 8, "latest_source_mass": 0.19101998209953308, "layer": 31, "max_source_mass": 0.20831164717674255, "mixer_index": 60, "normalized_entropy": 0.9090486661912383, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.10261696577072144, "median": 0.10389091819524765, "p05": 0.01371697522699833, "p95": 0.21163305640220642 }, { "label": "completed_group_1", "mean": 0.06039813905954361, "median": 0.06085003539919853, "p05": 0.026107093319296837, "p95": 0.09348798543214798 }, { "label": "completed_group_2", "mean": 0.06671924144029617, "median": 0.06656201928853989, "p05": 0.034684643149375916, "p95": 0.09819839894771576 }, { "label": "completed_group_3", "mean": 0.08377155661582947, "median": 0.0804532840847969, "p05": 0.05448959022760391, "p95": 0.12039553374052048 }, { "label": "completed_group_4", "mean": 0.07002593576908112, "median": 0.06737451255321503, "p05": 0.044841229915618896, "p95": 0.10655929893255234 }, { "label": "completed_group_5", "mean": 0.05768897384405136, "median": 0.056077782064676285, "p05": 0.039495617151260376, "p95": 0.08242568373680115 }, { "label": "completed_group_6", "mean": 0.20831164717674255, "median": 0.2021285891532898, "p05": 0.11652229726314545, "p95": 0.31706833839416504 }, { "label": "completed_group_7", "mean": 0.1594475358724594, "median": 0.1546279788017273, "p05": 0.06593266129493713, "p95": 0.2735573351383209 }, { "label": "current_group_partial", "mean": 0.19101998209953308, "median": 0.1713225245475769, "p05": 0.06408411264419556, "p95": 0.3850628137588501 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 1.969987154006958, "group": 8, "latest_source_mass": 0.16864418983459473, "layer": 31, "max_source_mass": 0.24503237009048462, "mixer_index": 61, "normalized_entropy": 0.8965797917640489, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.08120635151863098, "median": 0.061164144426584244, "p05": 0.0259944386780262, "p95": 0.23622018098831177 }, { "label": "completed_group_1", "mean": 0.07894632965326309, "median": 0.07658770680427551, "p05": 0.0496862456202507, "p95": 0.11596276611089706 }, { "label": "completed_group_2", "mean": 0.08615730702877045, "median": 0.08467075228691101, "p05": 0.050409119576215744, "p95": 0.12811316549777985 }, { "label": "completed_group_3", "mean": 0.07609779387712479, "median": 0.0722871869802475, "p05": 0.04099677503108978, "p95": 0.11950768530368805 }, { "label": "completed_group_4", "mean": 0.058728985488414764, "median": 0.056080836802721024, "p05": 0.03323862701654434, "p95": 0.09344407916069031 }, { "label": "completed_group_5", "mean": 0.05076971277594566, "median": 0.04946017265319824, "p05": 0.03047419898211956, "p95": 0.07563400268554688 }, { "label": "completed_group_6", "mean": 0.24503237009048462, "median": 0.24110376834869385, "p05": 0.10613486170768738, "p95": 0.3991503119468689 }, { "label": "completed_group_7", "mean": 0.1544169783592224, "median": 0.1400495171546936, "p05": 0.06199025735259056, "p95": 0.2995939254760742 }, { "label": "current_group_partial", "mean": 0.16864418983459473, "median": 0.15222924947738647, "p05": 0.0543956533074379, "p95": 0.3329208791255951 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.071648359298706, "group": 8, "latest_source_mass": 0.11520501971244812, "layer": 32, "max_source_mass": 0.20955611765384674, "mixer_index": 62, "normalized_entropy": 0.9428478002054053, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.10569609701633453, "median": 0.08689188957214355, "p05": 0.03173588588833809, "p95": 0.24571751058101654 }, { "label": "completed_group_1", "mean": 0.09922641515731812, "median": 0.09795533865690231, "p05": 0.06407676637172699, "p95": 0.14120742678642273 }, { "label": "completed_group_2", "mean": 0.08180510997772217, "median": 0.080217644572258, "p05": 0.054607540369033813, "p95": 0.11442367732524872 }, { "label": "completed_group_3", "mean": 0.10646799206733704, "median": 0.10510502755641937, "p05": 0.07415887713432312, "p95": 0.14216329157352448 }, { "label": "completed_group_4", "mean": 0.0826798751950264, "median": 0.08022409677505493, "p05": 0.05491747707128525, "p95": 0.11860857903957367 }, { "label": "completed_group_5", "mean": 0.06615597754716873, "median": 0.06485182791948318, "p05": 0.04477996751666069, "p95": 0.09238749742507935 }, { "label": "completed_group_6", "mean": 0.20955611765384674, "median": 0.206287682056427, "p05": 0.129981130361557, "p95": 0.2996031939983368 }, { "label": "completed_group_7", "mean": 0.13320739567279816, "median": 0.12465103715658188, "p05": 0.06303662061691284, "p95": 0.23065781593322754 }, { "label": "current_group_partial", "mean": 0.11520501971244812, "median": 0.10386130958795547, "p05": 0.03912060335278511, "p95": 0.22735844552516937 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 1.9781701564788818, "group": 8, "latest_source_mass": 0.16112206876277924, "layer": 32, "max_source_mass": 0.22251944243907928, "mixer_index": 63, "normalized_entropy": 0.9003040366848136, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.09446848928928375, "median": 0.074150450527668, "p05": 0.042592622339725494, "p95": 0.19443637132644653 }, { "label": "completed_group_1", "mean": 0.07056106626987457, "median": 0.06868506222963333, "p05": 0.04818308353424072, "p95": 0.0982733890414238 }, { "label": "completed_group_2", "mean": 0.08033448457717896, "median": 0.08047192543745041, "p05": 0.048296794295310974, "p95": 0.11253741383552551 }, { "label": "completed_group_3", "mean": 0.0658978745341301, "median": 0.06256154924631119, "p05": 0.04223886504769325, "p95": 0.0967884361743927 }, { "label": "completed_group_4", "mean": 0.058234501630067825, "median": 0.0563921295106411, "p05": 0.03529864549636841, "p95": 0.08846980333328247 }, { "label": "completed_group_5", "mean": 0.04316519945859909, "median": 0.042268507182598114, "p05": 0.029443470761179924, "p95": 0.05996086448431015 }, { "label": "completed_group_6", "mean": 0.2036968618631363, "median": 0.19645483791828156, "p05": 0.09398645162582397, "p95": 0.3395216464996338 }, { "label": "completed_group_7", "mean": 0.22251944243907928, "median": 0.2130982130765915, "p05": 0.11867167800664902, "p95": 0.35465875267982483 }, { "label": "current_group_partial", "mean": 0.16112206876277924, "median": 0.14023703336715698, "p05": 0.06008770316839218, "p95": 0.3247341811656952 } ], "sources": 9 }, { "branch": "output", "entropy_mean": 1.6036605834960938, "group": null, "latest_source_mass": 0.31639498472213745, "layer": null, "max_source_mass": 0.3412485122680664, "mixer_index": null, "normalized_entropy": 0.7298573846467136, "offset": null, "source_summaries": [ { "label": "embedding", "mean": 0.09759747982025146, "median": 0.09915056824684143, "p05": 0.025461740791797638, "p95": 0.16398483514785767 }, { "label": "completed_group_1", "mean": 0.04215940088033676, "median": 0.04050712287425995, "p05": 0.025720715522766113, "p95": 0.06398092210292816 }, { "label": "completed_group_2", "mean": 0.041145481169223785, "median": 0.03881259262561798, "p05": 0.023510465398430824, "p95": 0.0662468671798706 }, { "label": "completed_group_3", "mean": 0.029554028064012527, "median": 0.027420252561569214, "p05": 0.01627083122730255, "p95": 0.04924973472952843 }, { "label": "completed_group_4", "mean": 0.0289996936917305, "median": 0.027226522564888, "p05": 0.01560397818684578, "p95": 0.048351090401411057 }, { "label": "completed_group_5", "mean": 0.03647579252719879, "median": 0.0347352996468544, "p05": 0.020744025707244873, "p95": 0.058257561177015305 }, { "label": "completed_group_6", "mean": 0.06642461568117142, "median": 0.057809896767139435, "p05": 0.022837381809949875, "p95": 0.14078694581985474 }, { "label": "completed_group_7", "mean": 0.3412485122680664, "median": 0.3336990475654602, "p05": 0.0819513276219368, "p95": 0.6423859596252441 }, { "label": "completed_group_8", "mean": 0.31639498472213745, "median": 0.30713334679603577, "p05": 0.05841828137636185, "p95": 0.6156357526779175 } ], "sources": 9 } ], "mode": "uniform_output_only", "optimizer_state_after": "ed03e6fbd4a12d8b063dcb22e0437754285f54d585374cd52fbd534f05d24637", "optimizer_state_before": "ed03e6fbd4a12d8b063dcb22e0437754285f54d585374cd52fbd534f05d24637", "optimizer_state_unchanged": true, "positions": { "attention_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "5e6ecbcd78543fc42d4098a51169dca446d404ce384015643e84d2c78433c949", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.0933690071105957, 0.24537226557731628, 0.07698935270309448, 0.0523640513420105, 0.09380833059549332, 0.03647654876112938, 0.039120711386203766, 0.06371714919805527, 0.05156813934445381, 0.03728010877966881, 0.03983496502041817, 0.040284499526023865, 0.046871740370988846, 0.04314620420336723, 0.04033593460917473, 0.04471700266003609, 0.08397205173969269, 0.0528092160820961, 0.04443390294909477, 0.07560808956623077, 0.03708760067820549, 0.03599191829562187, 0.033304885029792786, 0.04061548039317131, 0.025487789884209633, 0.021565141156315804, 0.02342291921377182, 0.017244853079319, 0.024166326969861984, 0.016819356009364128, 0.021590016782283783, 0.026201345026493073 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 6.301439725575619e-06, "first_to_last_ratio": 0.836598579917906, "imbalance_abs_log_ratio": 0.17841091742000992, "last_quartile_mean": 7.5322142265576986e-06, "mean": 7.433581430404956e-06, "normalized": [ 0.7773002134099639, 0.9055587892136483, 0.8061625130839848, 0.7553921349061892, 0.9839302276183689, 0.9196554050293927, 0.8649825071810054, 0.768610403921652, 0.7038902292846156, 0.6349827054253709, 0.6010213694999877, 0.5764027609762332, 0.7429731749374235, 0.6985045282396805, 0.6611252376948986, 0.6208411261816975, 0.7329744097111522, 0.6946045779078276, 0.6969646990636194, 0.7216449105423735, 2.9451745132532094, 2.5024657030078794, 1.9956186121751214, 1.5830709023362801, 1.9068938397981738, 1.6772963842540904, 1.276661352303006, 0.8480176528865039, 0.9126088711887449, 0.6395649987165732, 0.4431599738929185, 0.40194527235841343 ], "peak_layer": 21, "peak_normalized": 2.9451745132532094, "population_cv": 0.5890096487368665, "rest_mean": 5.80005918756662e-06, "spike_contrast": 2.802488908488448, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.625460154173197e-05, "top_five_layers": [ 21, 22, 23, 25, 26 ], "top_five_spike_overlap": 4 }, "values": [ 5.778124432254117e-06, 6.731544999638572e-06, 5.992674687149702e-06, 5.615268946712604e-06, 7.314125468838029e-06, 6.836333341198042e-06, 6.429917903005844e-06, 5.713528025808046e-06, 5.232425337453606e-06, 4.720195647678338e-06, 4.467741291591665e-06, 4.284736860427074e-06, 5.5229515965038445e-06, 5.192390290176263e-06, 4.914528290100861e-06, 4.615073066815967e-06, 5.4486249609908555e-06, 5.1633996918099e-06, 5.1809438446071e-06, 5.364406206354033e-06, 2.1893194571021013e-05, 1.8602282580104657e-05, 1.4834593457635492e-05, 1.176788646262139e-05, 1.4175050637277309e-05, 1.2468319255276583e-05, 9.490166121395305e-06, 6.3038082771527115e-06, 6.783952358091483e-06, 4.754258497996489e-06, 3.294265752629144e-06, 2.9878929126425646e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 2.4883078140192083e-05, "first_to_last_ratio": 0.8278545594755969, "imbalance_abs_log_ratio": 0.18891779283288854, "last_quartile_mean": 3.005730638960813e-05, "mean": 2.9623082610896745e-05, "normalized": [ 0.7809815334326933, 0.9046248794749173, 0.8075923230382049, 0.7521123491775847, 0.9599168328777472, 0.898916536621374, 0.8532392322258369, 0.7625322562787574, 0.7033275492538346, 0.6347285738300478, 0.5999436224541148, 0.5757587388662117, 0.7443430270146784, 0.695529489803454, 0.6595663096410814, 0.6200506358258696, 0.7397139289573861, 0.6974643443511251, 0.7004244077698821, 0.721743176763463, 2.960216438672337, 2.5243781402151106, 2.0061273189191793, 1.5795020228936725, 1.9094994178766005, 1.675984518173989, 1.2731509943808097, 0.8475840656350129, 0.9131510116124364, 0.6440721835490694, 0.447818431493079, 0.40600570892043925 ], "peak_layer": 21, "peak_normalized": 2.960216438672337, "population_cv": 0.5926571528531915, "rest_mean": 2.306242192760153e-05, "spike_contrast": 2.8206339518416796, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.505065030069091e-05, "top_five_layers": [ 21, 22, 23, 25, 26 ], "top_five_spike_overlap": 4 }, "values": [ 2.313508048246149e-05, 2.6797777536557987e-05, 2.3923374101286754e-05, 2.2279886252363212e-05, 2.843569563992787e-05, 2.6628678824636154e-05, 2.5275576263084076e-05, 2.258855602121912e-05, 2.083473009406589e-05, 1.880261697806418e-05, 1.777217948983889e-05, 1.7055748685379513e-05, 2.2049734980100766e-05, 2.0603727534762584e-05, 1.9538387277862057e-05, 1.836781120800879e-05, 2.1912606825935654e-05, 2.0661043890868314e-05, 2.0748730094055645e-05, 2.1380257749115117e-05, 8.76907361089252e-05, 7.477986218873411e-05, 5.942767529631965e-05, 4.678971890825778e-05, 5.656525900121778e-05, 4.964782783645205e-05, 3.771465708268806e-05, 2.5108052795985714e-05, 2.7050347853219137e-05, 1.9079403500654735e-05, 1.3265762390801683e-05, 1.202714065584587e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.02206656220369041, "first_to_last_ratio": 0.827854555738895, "imbalance_abs_log_ratio": 0.18891779734660646, "last_quartile_mean": 0.02665511961095035, "mean": 0.026270045927958563, "normalized": [ 0.780981567453014, 0.9046248282127807, 0.8075923174907051, 0.7521123198054914, 0.9599168600183704, 0.8989165079424043, 0.853239239310957, 0.7625321952252507, 0.7033275518883858, 0.6347285851811321, 0.5999436237521429, 0.575758739660327, 0.7443430398131119, 0.6955294849676292, 0.6595663156843542, 0.6200506550895739, 0.7397139457418138, 0.6974643068926308, 0.7004243972519452, 0.7217431739242588, 2.9602165680150647, 2.5243782181619205, 2.0061272883989183, 1.579502031894164, 1.9094993545569388, 1.6759844712114402, 1.2731509728763686, 0.8475840259601726, 0.9131510767946985, 0.644072212359921, 0.44781842715829995, 0.4060056973058137 ], "peak_layer": 21, "peak_normalized": 2.9602165680150647, "population_cv": 0.5926571668834426, "rest_mean": 0.020451986299896682, "spike_contrast": 2.820633999729608, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.05768756791949272, "top_five_layers": [ 21, 22, 23, 25, 26 ], "top_five_spike_overlap": 4 }, "values": [ 0.020516421645879745, 0.023764535784721375, 0.021215487271547318, 0.01975802518427372, 0.02521705999970436, 0.02361457794904709, 0.022414634004235268, 0.020031755790114403, 0.018476447090506554, 0.016674349084496498, 0.015760546550154686, 0.015125208534300327, 0.019553925842046738, 0.018271591514348984, 0.0173268374055624, 0.0162887591868639, 0.019432319328188896, 0.018322419375181198, 0.018400181084871292, 0.018960226327180862, 0.07776502519845963, 0.06631553173065186, 0.05270105600357056, 0.04149359092116356, 0.05016263574361801, 0.04402818903326988, 0.03344573453068733, 0.022266071289777756, 0.023988520726561546, 0.016919806599617004, 0.011764210648834705, 0.010665788315236568 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 9.74586050972448e-07, "first_to_last_ratio": 0.4698352090962859, "imbalance_abs_log_ratio": 0.7553732647032987, "last_quartile_mean": 2.074314636502095e-06, "mean": 1.6101814175328855e-06, "normalized": [ 0.6786259026039465, 0.603176425245557, 0.5772428605710129, 0.5839292587279298, 0.6184502608288319, 0.6020128550631841, 0.590482213459805, 0.5881982127419906, 0.5149316297603258, 0.47825848175444796, 0.4675747116651752, 0.47792261383226664, 0.5514618379704384, 0.5552632104594527, 0.5635741234550302, 0.5658700564156129, 0.6059399750336795, 0.635006777207202, 0.6600719005441741, 0.6976252809204186, 2.9221316950822946, 2.761783816192785, 2.392519452837741, 2.0019542710166727, 2.3925409167537266, 2.1676593738842, 1.7020437740901044, 1.1036096739925716, 1.1045022622356275, 0.8035693344952884, 0.549066846213383, 0.48299999494512424 ], "peak_layer": 21, "peak_normalized": 2.9221316950822946, "population_cv": 0.7428539470469182, "rest_mean": 1.1646424211533116e-06, "spike_contrast": 3.4483476859837934, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.0160919979825845e-06, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 1.0927108178293565e-06, 9.712234714243095e-07, 9.294657274949714e-07, 9.40232041557465e-07, 9.958171176549513e-07, 9.693499123386573e-07, 9.507834874966647e-07, 9.471058319832082e-07, 8.291333415400004e-07, 7.700829200985027e-07, 7.52880112031562e-07, 7.695421118114609e-07, 8.879536039785307e-07, 8.940745033214625e-07, 9.07456580989674e-07, 9.111534495787055e-07, 9.756732879395713e-07, 1.0224761126664816e-06, 1.062835508491844e-06, 1.123303263739217e-06, 4.705162155005382e-06, 4.44697298007668e-06, 3.852390364045277e-06, 3.22350956594164e-06, 3.852424924843945e-06, 3.490324843369308e-06, 2.7405992568674264e-06, 1.7770117892723647e-06, 1.7784490182748414e-06, 1.293892410103581e-06, 8.840972327561758e-07, 7.777176165291166e-07 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 1.9441678432485787e-05, "first_to_last_ratio": 1.0045432560111063, "imbalance_abs_log_ratio": 0.004532966576767276, "last_quartile_mean": 1.9353749394213082e-05, "mean": 2.1282275241674142e-05, "normalized": [ 0.8660133984337955, 0.9822646513275938, 0.8885532078802768, 0.8316679175311164, 1.0413308716799239, 0.9719648702866424, 0.9114012210180789, 0.8149241494388278, 0.7704158954034577, 0.6974161075325436, 0.6578888986509813, 0.6288204871630333, 0.8173821721672502, 0.7584542417684855, 0.7151860246869496, 0.6677804337319789, 0.8168196106874187, 0.7690401753215117, 0.7737560507714033, 0.7920839995122299, 2.889411486464344, 2.399867443994061, 1.8607665385470271, 1.4017223538825623, 1.7608167481152943, 1.4634648010742246, 1.0986825701110414, 0.7505729964605627, 0.8167614913002589, 0.5750594557524572, 0.4225890554775739, 0.3871206738270936 ], "peak_layer": 21, "peak_normalized": 2.889411486464344, "population_cv": 0.5358337291896113, "rest_mean": 1.709472386816439e-05, "spike_contrast": 2.5677544134171617, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.389505265862681e-05, "top_five_layers": [ 21, 22, 23, 25, 26 ], "top_five_spike_overlap": 4 }, "values": [ 1.843073550844565e-05, 2.0904826669720933e-05, 1.8910433936980553e-05, 1.769978553056717e-05, 2.2161890228744596e-05, 2.068562389467843e-05, 1.9396691641304642e-05, 1.7343440049444325e-05, 1.6396203136537224e-05, 1.4842601558484603e-05, 1.4001372619532049e-05, 1.3382730685407296e-05, 1.73957523657009e-05, 1.6141631931532174e-05, 1.5220785826386418e-05, 1.4211886991688516e-05, 1.7383779777446762e-05, 1.636692468309775e-05, 1.6467289242427796e-05, 1.6857349692145362e-05, 6.149325054138899e-05, 5.1074639486614615e-05, 3.960134563385509e-05, 2.9831840947736055e-05, 3.74741866835393e-05, 3.1145860702963546e-05, 2.338246486033313e-05, 1.5973901099641807e-05, 1.738254286465235e-05, 1.2238573617651127e-05, 8.993656592792831e-06, 8.238808732130565e-06 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 1.4956453696868266e-05, "first_to_last_ratio": 1.1694726597418608, "imbalance_abs_log_ratio": 0.1565529290363499, "last_quartile_mean": 1.2789058018825017e-05, "mean": 1.5412839445616555e-05, "normalized": [ 0.9410119780727796, 1.0494377633850727, 0.9533461363714341, 0.8942925666204673, 1.1054924418284455, 1.0273886158237697, 0.9407003520718075, 0.8514441390090508, 0.8336623970789355, 0.7585381664715495, 0.7177133310683013, 0.6801097915167986, 0.88886014827631, 0.8218558963642031, 0.7846543254395936, 0.730005120976746, 0.908105487953464, 0.8463873713223805, 0.8583464100436482, 0.8747297590612398, 2.810834663962515, 2.246364495784679, 1.6787288541745102, 1.1598576954644273, 1.590109642187401, 1.2499498876136774, 0.9419154633408887, 0.6800642366459881, 0.7331510624631741, 0.5625391019248891, 0.45377879515841857, 0.42662390252343413 ], "peak_layer": 21, "peak_normalized": 2.810834663962515, "population_cv": 0.47699558742202136, "rest_mean": 1.285208445214839e-05, "spike_contrast": 2.275188629456653, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.9240916410344654e-05, "top_five_layers": [ 21, 22, 23, 25, 26 ], "top_five_spike_overlap": 4 }, "values": [ 1.4503666534437798e-05, 1.617481575522106e-05, 1.4693770935991779e-05, 1.3783587746729609e-05, 1.703877751424443e-05, 1.5834975783945993e-05, 1.4498863492917735e-05, 1.3123171811457723e-05, 1.284910467802547e-05, 1.1691226973198354e-05, 1.1062000339734368e-05, 1.0482423022040166e-05, 1.3699858754989691e-05, 1.2667132978094742e-05, 1.2093751138309017e-05, 1.1251451724092476e-05, 1.399648408551002e-05, 1.3045232662989292e-05, 1.3229555406724103e-05, 1.3482069334713742e-05, 4.3322943383827806e-05, 3.4622855309862643e-05, 2.5873978302115574e-05, 1.787670043995604e-05, 2.45081046159612e-05, 1.9265276932856068e-05, 1.4517591807816643e-05, 1.0481720892130397e-05, 1.1299939615128096e-05, 8.670324859849643e-06, 6.994019713602029e-06, 6.575485713256057e-06 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 5.010303812014172e-05, "first_to_last_ratio": 0.7825166271583448, "imbalance_abs_log_ratio": 0.24524010802892268, "last_quartile_mean": 6.40280811694538e-05, "mean": 6.0647469410923804e-05, "normalized": [ 0.7542362422864702, 0.8699737848114448, 0.7861215027554532, 0.7429077694197699, 0.9452668322436617, 0.8918822294120562, 0.8571828014367673, 0.7615142430830892, 0.6954672661434114, 0.6338508904354183, 0.5980899561253127, 0.5711141624548568, 0.7378101275889011, 0.6805091806350357, 0.6496067253078721, 0.6080743901731895, 0.7167869514381782, 0.6890245057517419, 0.6868515250228352, 0.7164486322853545, 2.8847543183739095, 2.4705878505698777, 2.004744811770676, 1.60125723206279, 1.9857222775321821, 1.7378130397055007, 1.32019295720206, 0.8899154194151095, 0.959140316936044, 0.6793479181386685, 0.4615471653719925, 0.4122569741103696 ], "peak_layer": 21, "peak_normalized": 2.8847543183739095, "population_cv": 0.5892906598174907, "rest_mean": 4.728915337182116e-05, "spike_contrast": 2.807882284930078, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001327823760220781, "top_five_layers": [ 21, 22, 23, 25, 26 ], "top_five_spike_overlap": 4 }, "values": [ 4.574251943267882e-05, 5.276170850265771e-05, 4.7676279791630805e-05, 4.505547622102313e-05, 5.7328041293658316e-05, 5.4090400226414204e-05, 5.198596772970632e-05, 4.618391176336445e-05, 4.217832974973135e-05, 3.8441452488768846e-05, 3.6272642319090664e-05, 3.463662869762629e-05, 4.4746317144017667e-05, 4.127115971641615e-05, 3.9397004002239555e-05, 3.687817297759466e-05, 4.3471314711496234e-05, 4.178759263595566e-05, 4.1655806853668764e-05, 4.345079651102424e-05, 0.00017495304928161204, 0.00014983490109443665, 0.00012158269964857027, 9.711219900054857e-05, 0.00012042903108522296, 0.00010539396316744387, 8.006636198842898e-05, 5.397111817728728e-05, 5.816943303216249e-05, 4.120073208468966e-05, 2.7991667593596503e-05, 2.5002342226798646e-05 ] } } }, "mlp_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "c0b8018c349b1bd09e5d17858eb312a2323da0e92de9651e2bb08ae4adf115c7", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.13583384454250336, 0.0415157787501812, 0.018140379339456558, 0.039518751204013824, 0.022877270355820656, 0.015532293356955051, 0.02314678393304348, 0.05432315170764923, 0.03743875026702881, 0.04006272554397583, 0.045583780854940414, 0.05995568633079529, 0.038651175796985626, 0.04559725895524025, 0.05632574483752251, 0.07952121645212173, 0.042337141931056976, 0.05044656619429588, 0.04418695345520973, 0.06310147047042847, 0.029010163620114326, 0.04586305841803551, 0.06033914536237717, 0.07656749337911606, 0.045884523540735245, 0.051604680716991425, 0.06502768397331238, 0.06792232394218445, 0.060306549072265625, 0.08025261014699936, 0.09663844108581543, 0.12230555713176727 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 6.241969458642416e-06, "first_to_last_ratio": 1.0529451337404454, "imbalance_abs_log_ratio": 0.051591127084412505, "last_quartile_mean": 5.928105139219042e-06, "mean": 6.639059385804558e-06, "normalized": [ 0.8675673045097559, 1.0124595149582853, 0.9035354501888456, 0.8438977762023232, 1.0968101130096073, 1.0190436004162458, 0.9440722623824992, 0.8341245971273976, 0.7302422990166911, 0.684091643661611, 0.652186254888741, 0.6290957211547357, 0.8086767136667771, 0.7557462804964336, 0.7048594180226722, 0.6692036814586189, 0.8143732996940146, 0.7617287677835181, 0.7613869739917988, 0.8464078714510895, 2.9570286465803957, 2.3011292370926366, 1.8226658757105683, 1.4363594359979903, 1.8957146750377063, 1.4390551547288326, 0.9578989514374155, 0.7576655999027994, 0.7163896500725841, 0.4985696380390154, 0.4512394516237545, 0.4267741396946395 ], "peak_layer": 21, "peak_normalized": 2.9570286465803957, "population_cv": 0.5418332581144877, "rest_mean": 5.308076037248678e-06, "spike_contrast": 2.60477984320151, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.3826369468006305e-05, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 5.759830855822656e-06, 6.721778845530935e-06, 5.998625510983402e-06, 5.602687451755628e-06, 7.28178747522179e-06, 6.765490979887545e-06, 6.267751814448275e-06, 5.537802735489095e-06, 4.848121989198262e-06, 4.541725047602085e-06, 4.3299032768118195e-06, 4.1766038521018345e-06, 5.368852725951001e-06, 5.0174444368167315e-06, 4.6796035348961595e-06, 4.442882982402807e-06, 5.406672698882176e-06, 5.057162525190506e-06, 5.054893335909583e-06, 5.619352123176213e-06, 1.9631888790172525e-05, 1.527733365946915e-05, 1.2100786989321932e-05, 9.536075594951399e-06, 1.2585762306116521e-05, 9.553972631692886e-06, 6.359548024192918e-06, 5.030186912335921e-06, 4.7561534302076325e-06, 3.310033434900106e-06, 2.995805516547989e-06, 2.833378857758362e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 2.4648492626511143e-05, "first_to_last_ratio": 1.041432768064779, "imbalance_abs_log_ratio": 0.040597426648966, "last_quartile_mean": 2.3667867367294093e-05, "mean": 2.6462837269036754e-05, "normalized": [ 0.8715015833474794, 1.0111864349696222, 0.9050914023383552, 0.8401105396598363, 1.0696060278658859, 0.9949880982782594, 0.9311300627404905, 0.8278899736331846, 0.7304349704040416, 0.683973228791121, 0.651590852809392, 0.6286876536882672, 0.8115311348681364, 0.7537789831433868, 0.7041333890460834, 0.6694221176645535, 0.8217192016156433, 0.7642476363854477, 0.7646314666121968, 0.8460858909512587, 2.9837342371865554, 2.323814873346749, 1.819873555577059, 1.4357861130849903, 1.8954329827677987, 1.4345878809014854, 0.9537284110278702, 0.7592925558776279, 0.7209884438582128, 0.5034515286793066, 0.45584850390560006, 0.43172026497410215 ], "peak_layer": 21, "peak_normalized": 2.9837342371865554, "population_cv": 0.5455973672601736, "rest_mean": 2.1112794726192346e-05, "spike_contrast": 2.6217782969170833, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.535306700039655e-05, "top_five_layers": [ 21, 22, 25, 23, 24 ], "top_five_spike_overlap": 5 }, "values": [ 2.306240457983222e-05, 2.6758862077258527e-05, 2.3951286493684165e-05, 2.2231708499020897e-05, 2.830481025739573e-05, 2.633020812936593e-05, 2.4640343326609582e-05, 2.1908317648922093e-05, 1.932938175741583e-05, 1.809987224987708e-05, 1.724294270388782e-05, 1.6636859072605148e-05, 2.1475416360772215e-05, 1.9947130567743443e-05, 1.863336729002185e-05, 1.7714808564051054e-05, 2.174502151319757e-05, 2.0224160834914073e-05, 2.0234318071743473e-05, 2.2389833247871138e-05, 7.895807357272133e-05, 6.149473483674228e-05, 4.815901775145903e-05, 3.79949742637109e-05, 5.01585345773492e-05, 3.796326564042829e-05, 2.5238359739887528e-05, 2.0093035345780663e-05, 1.9079399862675928e-05, 1.3322755876288284e-05, 1.206304477818776e-05, 1.1424543117755093e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.021858529886230826, "first_to_last_ratio": 1.0414327556742071, "imbalance_abs_log_ratio": 0.04059741475134538, "last_quartile_mean": 0.020988901844248176, "mean": 0.023467508959583938, "normalized": [ 0.8715016279015326, 1.0111864279634344, 0.9050913754636539, 0.8401105428974063, 1.0696060374562677, 0.9949880290072295, 0.9311300612909135, 0.8278899920845548, 0.7304349746525731, 0.6839731971874585, 0.6515908051540524, 0.6286876799330376, 0.8115311838640742, 0.7537790070794528, 0.7041333319264782, 0.6694221481966407, 0.8217191961120676, 0.7642475966402692, 0.7646314359287593, 0.8460859268913447, 2.9837342297480838, 2.3238149613060024, 1.8198735023931691, 1.435786099425061, 1.8954330124220395, 1.4345879112738935, 0.9537284009726806, 0.7592926092329372, 0.7209884484742808, 0.5034515002842458, 0.4558485232141042, 0.431720223622301 ], "peak_layer": 21, "peak_normalized": 2.9837342297480838, "population_cv": 0.5455973740702433, "rest_mean": 0.018723037645772652, "spike_contrast": 2.6217783130531735, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.04908765405416489, "top_five_layers": [ 21, 22, 25, 23, 24 ], "top_five_spike_overlap": 5 }, "values": [ 0.020451972261071205, 0.023730026558041573, 0.021240239962935448, 0.01971530169248581, 0.025100989267230034, 0.023349890485405922, 0.021851303055882454, 0.019428515806794167, 0.017141489312052727, 0.016051147133111954, 0.015291213057935238, 0.0147537337616086, 0.01904461532831192, 0.017689315602183342, 0.01652425527572632, 0.01570967026054859, 0.019283702597022057, 0.01793498732149601, 0.01794399507343769, 0.019855529069900513, 0.07002080976963043, 0.054534148424863815, 0.0427078977227211, 0.033694323152303696, 0.04448109120130539, 0.033666204661130905, 0.022381629794836044, 0.01781870611011982, 0.016919802874326706, 0.011814752593636513, 0.010697629302740097, 0.01013139821588993 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 9.632848616547562e-07, "first_to_last_ratio": 0.5786535675181149, "imbalance_abs_log_ratio": 0.5470513094636181, "last_quartile_mean": 1.6647004628111972e-06, "mean": 1.4041400167030815e-06, "normalized": [ 0.7711144428424656, 0.6895126786193331, 0.6616912879057165, 0.6718309159099509, 0.7071769123073398, 0.6792996149458727, 0.6611551346514561, 0.6464743161118918, 0.5485189060483381, 0.5248774784849891, 0.5246087541342054, 0.5387209927541015, 0.6181728407227788, 0.6119694293614197, 0.6071157121632319, 0.6246708044156085, 0.6896146950947737, 0.7127941337621986, 0.7353607448878692, 0.799920818819553, 2.862980327050019, 2.532510423540336, 2.2186136632241853, 1.8767681130483023, 2.503242706390978, 1.9789818203414797, 1.3784142299715025, 0.9999161033407515, 0.9215087502462188, 0.6309895915527636, 0.560502077018879, 0.5109715803314903 ], "peak_layer": 21, "peak_normalized": 2.862980327050019, "population_cv": 0.674471229748786, "rest_mean": 1.0404097544644039e-06, "spike_contrast": 3.23745852856398, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.3682834327919407e-06, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 1.082752646652807e-06, 9.681723440735368e-07, 9.291072160522162e-07, 9.433446734874451e-07, 9.929754014592618e-07, 9.538317726764944e-07, 9.283543818128237e-07, 9.077404570234648e-07, 7.701973459006695e-07, 7.370014714069839e-07, 7.366241447925859e-07, 7.564397037640447e-07, 8.68001222897874e-07, 8.592907647653192e-07, 8.524754662175837e-07, 8.7712527374606e-07, 9.68315589489066e-07, 1.000862766886712e-06, 1.0325494486096432e-06, 1.12320083189843e-06, 4.020025244244607e-06, 3.5559992284106556e-06, 3.1152442261372926e-06, 2.635245209603454e-06, 3.514903255563695e-06, 2.77876756626938e-06, 1.935486579895951e-06, 1.404022214046563e-06, 1.2939273119627615e-06, 8.859977356223681e-07, 7.870233957874007e-07, 7.174756433414586e-07 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 1.9277809997220174e-05, "first_to_last_ratio": 1.245761946384003, "imbalance_abs_log_ratio": 0.21974734784561992, "last_quartile_mean": 1.5474714132324152e-05, "mean": 1.9321152308293676e-05, "normalized": [ 0.9513826110453368, 1.080427222600904, 0.9797931167701956, 0.9145623229820717, 1.1425314617649631, 1.0593651087648366, 0.9798394360979595, 0.874152663597003, 0.791921264344892, 0.7413755803574347, 0.7038852154336824, 0.6767831836456668, 0.8785955532633346, 0.8105268533448942, 0.7529796075570242, 0.7101143647856416, 0.8937267219561669, 0.8315302207164805, 0.8340248742674734, 0.910773646747918, 2.9090121354226275, 2.2189078143759766, 1.6752810240746765, 1.2711410293904548, 1.6860931978884481, 1.2447821309636273, 0.842131321496011, 0.6837797086899101, 0.6374887656359873, 0.47116613314807326, 0.4303895907949233, 0.4115361180754039 ], "peak_layer": 21, "peak_normalized": 2.9090121354226275, "population_cv": 0.5008370455607027, "rest_mean": 1.5914593286989083e-05, "spike_contrast": 2.3699362178594607, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.771657102333847e-05, "top_five_layers": [ 21, 22, 25, 23, 24 ], "top_five_spike_overlap": 5 }, "values": [ 1.8381808331469074e-05, 2.0875098925898783e-05, 1.893073203973472e-05, 1.7670397937763482e-05, 2.2075024389778264e-05, 2.0468154616537504e-05, 1.8931626982521266e-05, 1.68896367540583e-05, 1.5300831364584155e-05, 1.4324230505735613e-05, 1.3599873454950284e-05, 1.3076230970909819e-05, 1.6975478501990438e-05, 1.5660312783438712e-05, 1.4548433682648465e-05, 1.3720227798330598e-05, 1.7267830116907135e-05, 1.606612204341218e-05, 1.6114321624627337e-05, 1.7597196347196586e-05, 5.620546653517522e-05, 4.2871855839621276e-05, 3.236835982534103e-05, 2.4559909434174187e-05, 3.257726348238066e-05, 2.405062514299061e-05, 1.6270947526209056e-05, 1.3211411896918435e-05, 1.2317017535679042e-05, 9.103472621063702e-06, 8.315622835652903e-06, 7.95135201769881e-06 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 1.4843270491837757e-05, "first_to_last_ratio": 1.403108227805812, "imbalance_abs_log_ratio": 0.3386899384201566, "last_quartile_mean": 1.0578849298781279e-05, "mean": 1.4256623828146076e-05, "normalized": [ 1.0175369474901643, 1.1325505215469958, 1.032782884671595, 0.9647314668936038, 1.1909934614936655, 1.0959875763729618, 0.9938872922894141, 0.9007223260935158, 0.8470201494992057, 0.7955192033689217, 0.7574609142714921, 0.7198840825371114, 0.9367342778602812, 0.8645177148280875, 0.8051665958671916, 0.7624881150904066, 0.9766348255677715, 0.903024032951712, 0.9125254635516572, 0.9813276154290733, 2.8164200576739487, 2.0961437663296256, 1.5088115878867194, 1.050885349790649, 1.4582305589652822, 1.0598247505286014, 0.7470886244608695, 0.6356012933803904, 0.6114482992907498, 0.4947107735908344, 0.46318935110904286, 0.46615011931845923 ], "peak_layer": 21, "peak_normalized": 2.8164200576739487, "population_cv": 0.45294573026415497, "rest_mean": 1.2181233597840657e-05, "spike_contrast": 2.0904065969401686, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.5463731071795336e-05, "top_five_layers": [ 21, 22, 23, 25, 5 ], "top_five_spike_overlap": 4 }, "values": [ 1.4506641491607297e-05, 1.6146346752066165e-05, 1.47239970829105e-05, 1.3753813618677668e-05, 1.6979545762296766e-05, 1.5625082596670836e-05, 1.4169477253744844e-05, 1.2841259376727976e-05, 1.2075647646270227e-05, 1.1341418030497152e-05, 1.0798835319292266e-05, 1.026311656460166e-05, 1.3354668226384092e-05, 1.2325103853072505e-05, 1.1478957276267465e-05, 1.0870506230276078e-05, 1.3923515325586777e-05, 1.2874073945567943e-05, 1.30095322674606e-05, 1.3990418665343896e-05, 4.015264130430296e-05, 2.98839331662748e-05, 2.151055923604872e-05, 1.498207711847499e-05, 2.0789444533875212e-05, 1.5109522792045027e-05, 1.0650961485225707e-05, 9.061528544407338e-06, 8.717188393347897e-06, 7.052905402815668e-06, 6.603516339964699e-06, 6.645726898568682e-06 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 4.9593145377002656e-05, "first_to_last_ratio": 0.9907700333176028, "imbalance_abs_log_ratio": 0.009272826760126446, "last_quartile_mean": 5.005515276934602e-05, "mean": 5.409445373061317e-05, "normalized": [ 0.8395306469845508, 0.9740368143996314, 0.8813342332061235, 0.8294224179345759, 1.0531990104229512, 0.9885788559803391, 0.9440754916639701, 0.824126430354638, 0.7290515227289374, 0.6845574392360309, 0.6490317248157998, 0.6251865992466682, 0.8060147688627204, 0.7390877917709052, 0.6900387742758448, 0.6598528287114513, 0.7973538775065009, 0.7589838597043641, 0.7481952386908317, 0.8371002145272011, 2.891504573998991, 2.339382198994318, 1.8315580014046786, 1.476166756028569, 1.9444352769925548, 1.4796997906646798, 0.9997051510127529, 0.8002740415323617, 0.7599857851242594, 0.5194504801730139, 0.4633341828635408, 0.4357452201862441 ], "peak_layer": 21, "peak_normalized": 2.891504573998991, "population_cv": 0.5418898499099017, "rest_mean": 4.310917884814208e-05, "spike_contrast": 2.630876790659282, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00011341493809595704, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 4.5413951738737524e-05, 5.2689989388454705e-05, 4.767529389937408e-05, 4.486715261009522e-05, 5.6972225138451904e-05, 5.3476633183890954e-05, 5.1069248002022505e-05, 4.458066905499436e-05, 3.943764386349358e-05, 3.7030760722700506e-05, 3.5109016607748345e-05, 3.381912756594829e-05, 4.36009286204353e-05, 3.9980550354812294e-05, 3.732727054739371e-05, 3.569437831174582e-05, 4.313242243370041e-05, 4.105681728105992e-05, 4.047321272082627e-05, 4.5282478822628036e-05, 0.00015641436039004475, 0.00012654760212171823, 9.907712956191972e-05, 7.985243428265676e-05, 0.00010518316412344575, 8.004355186130852e-05, 5.4078504035715014e-05, 4.329038711148314e-05, 4.111101588932797e-05, 2.8099389965063892e-05, 2.5063809516723268e-05, 2.357139965170063e-05 ] } } }, "post_attention_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "90a633f12d984935028fe5d7212be48e7da7b4a0806751939a551938d16dc540", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.0933690071105957, 0.35791492462158203, 0.3940892517566681, 0.4057151973247528, 0.09380833059549332, 0.11286329478025436, 0.13231655955314636, 0.1621972620487213, 0.05156813934445381, 0.08533531427383423, 0.12003090232610703, 0.16063299775123596, 0.046871740370988846, 0.079384446144104, 0.11459959298372269, 0.15625107288360596, 0.08397205173969269, 0.11654024571180344, 0.15294702351093292, 0.21572710573673248, 0.03708760067820549, 0.06553087383508682, 0.09390194714069366, 0.12852425873279572, 0.025487789884209633, 0.06059965491294861, 0.08860157430171967, 0.12100546807050705, 0.024166326969861984, 0.06877554953098297, 0.12682415544986725, 0.2025238573551178 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 6.301456380697346e-06, "first_to_last_ratio": 0.8366045634852721, "imbalance_abs_log_ratio": 0.1784037651894039, "last_quartile_mean": 7.532180262614929e-06, "mean": 7.433566544534642e-06, "normalized": [ 0.7772975489047199, 0.9055563203739933, 0.8061699390522095, 0.75538905948044, 0.9839501221912754, 0.9196750485409326, 0.8649787335894211, 0.7686069267417784, 0.7038868060258207, 0.6349933367440544, 0.6010189637412672, 0.5764014070656202, 0.7429593078680798, 0.6985048258620252, 0.6611261945626958, 0.620849159837693, 0.732984992560827, 0.6946063970918009, 0.6969662782743665, 0.7216519837377928, 2.9452413411808753, 2.502428870646664, 1.9955997290470113, 1.583028680726461, 1.9068801623840628, 1.6773127121438207, 1.2766334437662124, 0.8480196569336151, 0.9126077011389664, 0.6395629148416826, 0.4431616260155856, 0.4019498089282282 ], "peak_layer": 21, "peak_normalized": 2.9452413411808753, "population_cv": 0.5890102634302898, "rest_mean": 5.800059903372636e-06, "spike_contrast": 2.8024714705718403, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.6254502406809478e-05, "top_five_layers": [ 21, 22, 23, 25, 26 ], "top_five_spike_overlap": 4 }, "values": [ 5.778093054686906e-06, 6.73151316732401e-06, 5.992717888148036e-06, 5.615234840661287e-06, 7.314258709811838e-06, 6.83646567267715e-06, 6.429876975744264e-06, 5.713490736525273e-06, 5.232389412412886e-06, 4.720265224023024e-06, 4.4677144614979625e-06, 4.284718215785688e-06, 5.522837454918772e-06, 5.192382104723947e-06, 4.914525561616756e-06, 4.615123543771915e-06, 5.4486927183461376e-06, 5.163402875041356e-06, 5.1809452088491526e-06, 5.364448043110315e-06, 2.1893647499382496e-05, 1.860197153291665e-05, 1.483442338212626e-05, 1.1767549040087033e-05, 1.4174920579534955e-05, 1.2468415661714971e-05, 9.489939657214563e-06, 6.303810550889466e-06, 6.7839300754712895e-06, 4.75423348689219e-06, 3.29427143697103e-06, 2.9879206522309687e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 2.4883182732082787e-05, "first_to_last_ratio": 0.8278604440222168, "imbalance_abs_log_ratio": 0.1889106846692161, "last_quartile_mean": 3.005721907811676e-05, "mean": 2.9623073061202376e-05, "normalized": [ 0.7809766272241224, 0.904628671157397, 0.807604066022385, 0.7521086003474774, 0.9599214406431313, 0.8989321775283157, 0.8532430073429444, 0.7625317652452033, 0.7033208372817493, 0.6347317872692634, 0.5999431404107706, 0.575757573577019, 0.7443337492767284, 0.6955254157101299, 0.6595627151900221, 0.6200582656558817, 0.7397196324203087, 0.6974642621727606, 0.7004234054785605, 0.7217455585911596, 2.9602660253160598, 2.524357830866588, 2.0061225620472483, 1.579475514112009, 1.9094879981705302, 1.675996356891378, 1.2731332290848365, 0.8475781370211712, 0.9131469462700484, 0.6440722683719267, 0.44782090922813156, 0.40600952407474183 ], "peak_layer": 21, "peak_normalized": 2.9602660253160598, "population_cv": 0.5926590116051075, "rest_mean": 2.3062429203559145e-05, "spike_contrast": 2.8206287082037655, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.505054989247583e-05, "top_five_layers": [ 21, 22, 23, 25, 26 ], "top_five_spike_overlap": 4 }, "values": [ 2.3134927687351592e-05, 2.679788121895399e-05, 2.3923714252305217e-05, 2.2279768018051982e-05, 2.843582296918612e-05, 2.662913357198704e-05, 2.527567994548008e-05, 2.2588534193346277e-05, 2.083452454826329e-05, 1.8802706108544953e-05, 1.7772159480955452e-05, 1.7055708667612635e-05, 2.2049453036743216e-05, 2.0603600205504335e-05, 1.9538274500519037e-05, 1.8368031305726618e-05, 2.191276871599257e-05, 2.0661034795921296e-05, 2.0748693714267574e-05, 2.138032141374424e-05, 8.76921767485328e-05, 7.47792364563793e-05, 5.9427515225252137e-05, 4.678891855292022e-05, 5.656490247929469e-05, 4.9648162530502304e-05, 3.7714118661824614e-05, 2.5107869078055955e-05, 2.7050218704971485e-05, 1.9079399862675928e-05, 1.3265831512399018e-05, 1.2027249795210082e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.022066655103117228, "first_to_last_ratio": 0.8278604273069176, "imbalance_abs_log_ratio": 0.18891070486017889, "last_quartile_mean": 0.026655042776837945, "mean": 0.02627003754605539, "normalized": [ 0.7809765697582473, 0.9046286620381037, 0.807604061580307, 0.7521085891676671, 0.9599214914272164, 0.8989321808788358, 0.8532429858370449, 0.7625317294862717, 0.7033208986296787, 0.6347317656611623, 0.5999431061365093, 0.57575757619401, 0.7443337052971855, 0.6955254526609387, 0.6595626973257241, 0.6200582978249476, 0.7397195704479738, 0.6974643167191428, 0.700423415369905, 0.7217456022261184, 2.960266010712984, 2.524357752471682, 2.0061225398001983, 1.5794755924221078, 1.9094880519790167, 1.6759963505687836, 1.2731332277273864, 0.847578127766823, 0.9131469721163114, 0.6440722760545624, 0.44782090986737505, 0.40600951384577966 ], "peak_layer": 21, "peak_normalized": 2.960266010712984, "population_cv": 0.5926590078266182, "rest_mean": 0.02045199292263499, "spike_contrast": 2.8206287148026856, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.05768747851252556, "top_five_layers": [ 21, 22, 23, 25, 26 ], "top_five_spike_overlap": 4 }, "values": [ 0.020516283810138702, 0.023764628916978836, 0.021215789020061493, 0.019757920876145363, 0.025217173621058464, 0.023614982143044472, 0.02241472527384758, 0.02003173716366291, 0.018476266413927078, 0.016674427315592766, 0.015760527923703194, 0.015125173144042492, 0.01955367438495159, 0.01827147975564003, 0.017326736822724342, 0.01628895476460457, 0.019432460889220238, 0.01832241378724575, 0.018400149419903755, 0.01896028406918049, 0.0777662992477417, 0.06631497293710709, 0.052700914442539215, 0.04149288311600685, 0.05016232281923294, 0.04402848705649376, 0.03344525769352913, 0.022265909239649773, 0.023988405242562294, 0.016919802874326706, 0.01176427211612463, 0.010665885172784328 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 9.745850135800538e-07, "first_to_last_ratio": 0.46983166565059253, "imbalance_abs_log_ratio": 0.7553808066222247, "last_quartile_mean": 2.074328072865228e-06, "mean": 1.6101727560169365e-06, "normalized": [ 0.6786195271378114, 0.6032026872296619, 0.5772551444002313, 0.5839160193822234, 0.6184416553181962, 0.6019946294072027, 0.590489661433786, 0.5882195576869884, 0.5149255034282336, 0.4782557943248373, 0.467532745483789, 0.47793478702560815, 0.5514496242657089, 0.5552487225288281, 0.5636254844325217, 0.5658572141625743, 0.6059313022271344, 0.634980397600398, 0.6600549756772345, 0.6976184428136875, 2.9221397885690084, 2.7617667588826444, 2.3925150950798675, 2.001910109046701, 2.3925631067291526, 2.1677302015552202, 1.702065779979614, 1.103576212786341, 1.1045094745220083, 0.8035674438204516, 0.5490768250131973, 0.4830253280491365 ], "peak_layer": 21, "peak_normalized": 2.9221397885690084, "population_cv": 0.742856944866144, "rest_mean": 1.1646382610571756e-06, "spike_contrast": 3.4483316949884104, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.016059028799645e-06, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 1.0926946742984e-06, 9.712605333334068e-07, 9.294805067838752e-07, 9.402056662111136e-07, 9.957979045793763e-07, 9.6931535153999e-07, 9.507903655503469e-07, 9.471351063439215e-07, 8.291190169984475e-07, 7.700744504290924e-07, 7.528084893237974e-07, 7.695575732213911e-07, 8.879291613084206e-07, 8.940463658291264e-07, 9.075343996300944e-07, 9.111278700402181e-07, 9.756540748639964e-07, 1.0224281368209631e-06, 1.0628025393089047e-06, 1.1232862107135588e-06, 4.7051498768269084e-06, 4.44692159362603e-06, 3.852362624456873e-06, 3.2234211175818928e-06, 3.852439931506524e-06, 3.4904201129393186e-06, 2.7406199478718918e-06, 1.776948352016916e-06, 1.7784510646379204e-06, 1.2938824056618614e-06, 8.841085445965291e-07, 7.77754223690863e-07 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 1.944173368428892e-05, "first_to_last_ratio": 1.0045473264258957, "imbalance_abs_log_ratio": 0.004537018574049067, "last_quartile_mean": 1.935372597472451e-05, "mean": 2.128230337916648e-05, "normalized": [ 0.8660091765663459, 0.98227001929493, 0.8885606655444147, 0.8316680145507837, 1.0413331701202786, 0.971967174966732, 0.9113988194736778, 0.8149243540651301, 0.7704131674393653, 0.6974165102518931, 0.6578902510600761, 0.6288166216248912, 0.8173782710042303, 0.7584522133755768, 0.7151833270078949, 0.6677830123722243, 0.8168181888858115, 0.769040867959229, 0.7737553696610424, 0.792085772787997, 2.8894387772767613, 2.3998581173020517, 1.8607697194055655, 1.4017190476722055, 1.7608110013433533, 1.463469361904099, 1.0986688953855233, 0.7505691836264532, 0.8167622917844157, 0.5750566869270645, 0.4225898215478227, 0.3871221278121598 ], "peak_layer": 21, "peak_normalized": 2.8894387772767613, "population_cv": 0.5358360189712266, "rest_mean": 1.7094736937940104e-05, "spike_contrast": 2.5677588559065727, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.3895162161788906e-05, "top_five_layers": [ 21, 22, 23, 25, 26 ], "top_five_spike_overlap": 4 }, "values": [ 1.8430670024827123e-05, 2.090496855089441e-05, 1.891061765491031e-05, 1.769981099641882e-05, 2.216196844528895e-05, 2.0685700292233378e-05, 1.9396666175452992e-05, 1.734346733428538e-05, 1.6396166756749153e-05, 1.4842629752820358e-05, 1.4001419913256541e-05, 1.338266611128347e-05, 1.7395692339050584e-05, 1.6141610103659332e-05, 1.5220748537103646e-05, 1.421196066075936e-05, 1.7383772501489148e-05, 1.636696106288582e-05, 1.646729651838541e-05, 1.685740971879568e-05, 6.149391265353188e-05, 5.107450851937756e-05, 3.9601465687155724e-05, 2.9831810024916194e-05, 3.747411392396316e-05, 3.1145998946158215e-05, 2.3382204744848423e-05, 1.597384107299149e-05, 1.738258288241923e-05, 1.2238530871400144e-05, 8.993684787128586e-06, 8.238850568886846e-06 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 1.4956979725866404e-05, "first_to_last_ratio": 1.169562847710864, "imbalance_abs_log_ratio": 0.15663004455576848, "last_quartile_mean": 1.2788521587481227e-05, "mean": 1.541278261640855e-05, "normalized": [ 0.9408618469899721, 1.0496825080352417, 0.9532101539455953, 0.8942444670679459, 1.1056651660005725, 1.0275204537426434, 0.940834643786764, 0.8513964125501008, 0.833560198649658, 0.7584613600142788, 0.7176599187217451, 0.6801438100509339, 0.8885706223936058, 0.821914749293628, 0.7845320012285661, 0.730019555427945, 0.9080090518548033, 0.8465156504161461, 0.8585500878753176, 0.8747557618385634, 2.811136769002808, 2.246521481438981, 1.6786195040437275, 1.1597356925195756, 1.5899541742292345, 1.2494614161771285, 0.9421228718328318, 0.6801980394299975, 0.7332081130968231, 0.5625353341870779, 0.45386960158003625, 0.42652858257775145 ], "peak_layer": 21, "peak_normalized": 2.811136769002808, "population_cv": 0.4770261475916537, "rest_mean": 1.2851995810099956e-05, "spike_contrast": 2.2752132666815377, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.9241031370474957e-05, "top_five_layers": [ 21, 22, 23, 25, 26 ], "top_five_spike_overlap": 4 }, "values": [ 1.4501299119729083e-05, 1.61785283125937e-05, 1.4691620890516788e-05, 1.3782795576844364e-05, 1.7041376850102097e-05, 1.583694938744884e-05, 1.4500879842671566e-05, 1.3122387827024795e-05, 1.2847482139477506e-05, 1.1690000064845663e-05, 1.1061136319767684e-05, 1.0482908692210913e-05, 1.3695345842279494e-05, 1.2667993360082619e-05, 1.2091821190551855e-05, 1.1251632713538129e-05, 1.3994946129969321e-05, 1.3047161701251753e-05, 1.3232645869720727e-05, 1.3482420399668626e-05, 4.332743992563337e-05, 3.462514723651111e-05, 2.5872197511489503e-05, 1.7874754121294245e-05, 2.4505618057446554e-05, 1.9257677195128053e-05, 1.452073502150597e-05, 1.0483744517841842e-05, 1.1300777259748429e-05, 8.670234819874167e-06, 6.995393505349057e-06, 6.573992322955746e-06 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 5.010748100175988e-05, "first_to_last_ratio": 0.7826747046000525, "imbalance_abs_log_ratio": 0.24503811682307108, "last_quartile_mean": 6.402082590284408e-05, "mean": 6.0648911926364235e-05, "normalized": [ 0.7539852642456891, 0.8697480666287902, 0.786134416794645, 0.743112581131518, 0.9455673045080062, 0.8917968931204828, 0.8576296308462544, 0.7615400991514346, 0.6954606220709333, 0.6337862675086987, 0.5983485390597801, 0.5712770523128787, 0.7377510699121531, 0.6804594037629287, 0.6496227063557205, 0.6081313085576096, 0.7167360717736608, 0.6891680354991823, 0.6864821812292143, 0.7162888892744959, 2.8844812791075043, 2.47105766951999, 2.0052967319906334, 1.6013597497147145, 1.9853427350654518, 1.737596552387888, 1.3200243128865916, 0.8896987044366073, 0.9590050336354964, 0.6792431033557431, 0.46161020816722115, 0.4122575159880818 ], "peak_layer": 21, "peak_normalized": 2.8844812791075043, "population_cv": 0.5893001364175143, "rest_mean": 4.728921865332975e-05, "spike_contrast": 2.8080661804590905, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013279125560075044, "top_five_layers": [ 21, 22, 23, 25, 26 ], "top_five_spike_overlap": 4 }, "values": [ 4.572838588501327e-05, 5.274927389109507e-05, 4.767819700646214e-05, 4.506896948441863e-05, 5.73476281715557e-05, 5.408651122706942e-05, 5.2014303946634755e-05, 4.618657840183005e-05, 4.217893001623452e-05, 3.843844751827419e-05, 3.628918784670532e-05, 3.4647331631276757e-05, 4.474379966268316e-05, 4.126912244828418e-05, 3.939891030313447e-05, 3.68825021723751e-05, 4.3469262891449034e-05, 4.179729148745537e-05, 4.163439734838903e-05, 4.3442141759442165e-05, 0.00017494065104983747, 0.00014986695896368474, 0.00012161906488472596, 9.71207264228724e-05, 0.00012040887668263167, 0.00010538334026932716, 8.005803829291835e-05, 5.395925836637616e-05, 5.816261182189919e-05, 4.119535515201278e-05, 2.7996156859444454e-05, 2.500296977814287e-05 ] } } }, "post_mlp_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "89f2226456bfe96e245d2aad0f699d32c676f9f53099789cd870253d002402f5", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.16915453970432281, 0.374446302652359, 0.39966297149658203, 0.4194087088108063, 0.09892481565475464, 0.11664740741252899, 0.13951337337493896, 0.18593810498714447, 0.06607726216316223, 0.10425819456577301, 0.14551182091236115, 0.19865906238555908, 0.06484340131282806, 0.10430748760700226, 0.14720691740512848, 0.20439592003822327, 0.10052042454481125, 0.14085541665554047, 0.1745898574590683, 0.2420409768819809, 0.049456026405096054, 0.08509506285190582, 0.11783231794834137, 0.15649692714214325, 0.05533855780959129, 0.08507220447063446, 0.11942370235919952, 0.1559232622385025, 0.06628894805908203, 0.12384890764951706, 0.19928614795207977, 0.30097496509552 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 6.2419582604889e-06, "first_to_last_ratio": 1.0529455063592608, "imbalance_abs_log_ratio": 0.051591480966811015, "last_quartile_mean": 5.928092406293217e-06, "mean": 6.639059101587463e-06, "normalized": [ 0.8675586426895757, 1.0124664078767984, 0.9035421329570779, 0.8438917162074596, 1.096808584561565, 1.0190363834915506, 0.9440810017586895, 0.8341125775836542, 0.7302432207230373, 0.6840977690694656, 0.6521862828087338, 0.6291023236784927, 0.8086905844281629, 0.7557389153085153, 0.7048532150840923, 0.6692041895773945, 0.8143629232027878, 0.7617198274493953, 0.7613932397001602, 0.8464072912239047, 2.957035896728624, 2.301175364749489, 1.822643624123208, 1.4363556617261604, 1.895703385897998, 1.4390576821816614, 0.9578963896063162, 0.7576608376356524, 0.7163911191519227, 0.49857188549466974, 0.4512367653590111, 0.42677415796477447 ], "peak_layer": 21, "peak_normalized": 2.957035896728624, "population_cv": 0.5418360268229179, "rest_mean": 5.308071860310048e-06, "spike_contrast": 2.6047857995045867, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.3826390204485506e-05, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 5.759773102909094e-06, 6.721824320266023e-06, 5.998669621476438e-06, 5.602646979241399e-06, 7.28177701603272e-06, 6.765442776668351e-06, 6.267809567361837e-06, 5.537722699955339e-06, 4.848127900913823e-06, 4.541765520116314e-06, 4.3299032768118195e-06, 4.17664750784752e-06, 5.36894458491588e-06, 5.017395324102836e-06, 4.679562152887229e-06, 4.442886165634263e-06, 5.406603577284841e-06, 5.05710295328754e-06, 5.054934717918513e-06, 5.619348030450055e-06, 1.9631936083897017e-05, 1.5277639249688946e-05, 1.2100638741685543e-05, 9.536050129099749e-06, 1.2585686818056274e-05, 9.553989002597518e-06, 6.359530743793584e-06, 5.030155080021359e-06, 4.756162979902001e-06, 3.3100482141890097e-06, 2.995787554027629e-06, 2.833378857758362e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 2.4648483076816774e-05, "first_to_last_ratio": 1.0414370018621426, "imbalance_abs_log_ratio": 0.04060149199901786, "last_quartile_mean": 2.3667761979595525e-05, "mean": 2.6462811831606814e-05, "normalized": [ 0.8715039333066501, 1.0111786773030726, 0.9050989399036203, 0.8401052983115095, 1.0696100804799027, 0.9949961346823256, 0.9311295143021071, 0.8278858203389484, 0.7304403466909128, 0.6839769794524219, 0.651593610017078, 0.6286941694466056, 0.811534252031446, 0.7537748273473861, 0.7041346157962559, 0.6694198054323341, 0.8217166233538702, 0.7642455527795864, 0.7646342637428244, 0.84608780405412, 2.9837351806549774, 2.3238267303782894, 1.8198649942984655, 1.435786255960941, 1.8954130836840895, 1.4345797741171293, 0.9537325584670124, 0.7592921172103865, 0.7209863186693676, 0.5034518064103666, 0.4558492514099402, 0.4317206799660563 ], "peak_layer": 21, "peak_normalized": 2.9837351806549774, "population_cv": 0.5455964820740874, "rest_mean": 2.1112789639759017e-05, "spike_contrast": 2.6217725185565164, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.5352931667584926e-05, "top_five_layers": [ 21, 22, 25, 23, 24 ], "top_five_spike_overlap": 5 }, "values": [ 2.3062444597599097e-05, 2.6758631065604277e-05, 2.395146293565631e-05, 2.2231548427953385e-05, 2.8304890292929485e-05, 2.6330395485274494e-05, 2.4640305127832107e-05, 2.1908186681685038e-05, 1.9329505448695272e-05, 1.809995410440024e-05, 1.724299909255933e-05, 1.6637015505693853e-05, 2.1475478206411935e-05, 1.994700141949579e-05, 1.863338184193708e-05, 1.7714730347506702e-05, 2.1744932382716797e-05, 2.0224086256348528e-05, 2.023437264142558e-05, 2.2389862351701595e-05, 7.895802264101803e-05, 6.149498949525878e-05, 4.81587449030485e-05, 3.799494152190164e-05, 5.015795977669768e-05, 3.79630146198906e-05, 2.5238445232389495e-05, 2.0093004422960803e-05, 1.9079325284110382e-05, 1.3322750419320073e-05, 1.2063052963640075e-05, 1.1424543117755093e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.0218585217371583, "first_to_last_ratio": 1.0414370173790213, "imbalance_abs_log_ratio": 0.04060150689850628, "last_quartile_mean": 0.020988808129914105, "mean": 0.023467486258596182, "normalized": [ 0.8715039789929538, 1.0111786752781364, 0.9050989975542326, 0.8401053233485017, 1.0696100882364912, 0.9949961349147988, 0.93112953332467, 0.8278857925412914, 0.7304403641373193, 0.6839770336730604, 0.6515936181735539, 0.6286941615634386, 0.8115341912856134, 0.7537748152171392, 0.70413456866048, 0.6694198590155184, 0.8217165780257787, 0.7642455579300145, 0.7646342392397752, 0.8460877771693306, 2.983735211115682, 2.3238265750348694, 1.8198651032978919, 1.4357862183743275, 1.8954130982069366, 1.4345797744477677, 0.953732498402702, 0.7592921531557547, 0.7209863679178145, 0.5034518285493168, 0.45584924197322396, 0.43172064124161497 ], "peak_layer": 21, "peak_normalized": 2.983735211115682, "population_cv": 0.545596478166527, "rest_mean": 0.018723033092640066, "spike_contrast": 2.621772504053058, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.04908753335475922, "top_five_layers": [ 21, 22, 25, 23, 24 ], "top_five_spike_overlap": 5 }, "values": [ 0.02045200765132904, 0.023729821667075157, 0.021240398287773132, 0.019715160131454468, 0.025101060047745705, 0.023350058123469353, 0.021851269528269768, 0.019428398460149765, 0.01714159920811653, 0.016051221638917923, 0.015291264280676842, 0.014753871597349644, 0.0190446674823761, 0.01768920011818409, 0.016524268314242363, 0.01570960134267807, 0.01928362250328064, 0.017934922128915787, 0.01794404350221157, 0.019855553284287453, 0.07002076506614685, 0.054534368216991425, 0.042707659304142, 0.03369429334998131, 0.0444805808365345, 0.033665981143713, 0.022381704300642014, 0.01781867817044258, 0.016919737681746483, 0.011814748868346214, 0.01069763582199812, 0.01013139821588993 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 9.632890254351878e-07, "first_to_last_ratio": 0.5786689197715291, "imbalance_abs_log_ratio": 0.5470247788231377, "last_quartile_mean": 1.6646634932726556e-06, "mean": 1.4041316429569406e-06, "normalized": [ 0.7711311054221659, 0.6895686897971351, 0.6617110223575132, 0.6718209558481308, 0.7071836396052803, 0.6793116007113885, 0.661156567603486, 0.64642817498915, 0.5484996686905381, 0.5248870859444074, 0.5245894146676577, 0.5387402772351952, 0.6181933682031117, 0.6120226705693413, 0.6070899826342355, 0.6246342491771548, 0.6895792153692765, 0.7128092340510411, 0.7353607581593903, 0.7998615452071496, 2.8630780429312357, 2.532557265188057, 2.218595641401312, 1.8768170355595688, 2.503138129121275, 1.9789539489810566, 1.3784506264950211, 0.9998694386271998, 0.9215078494903352, 0.6310098716039221, 0.560468296772398, 0.5109746275858701 ], "peak_layer": 21, "peak_normalized": 2.8630780429312357, "population_cv": 0.6744764975921241, "rest_mean": 1.040399863709522e-06, "spike_contrast": 3.23748913123024, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.3682832508930006e-06, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 1.0827695859916275e-06, 9.682452173365164e-07, 9.29129384985572e-07, 9.433250625079381e-07, 9.929789257512311e-07, 9.538429139865912e-07, 9.283508575208543e-07, 9.076702554011717e-07, 7.701657409597829e-07, 7.370105663540016e-07, 7.365925966951181e-07, 7.564622706013324e-07, 8.680248697601201e-07, 8.593603979534237e-07, 8.524342547389097e-07, 8.770687145442935e-07, 9.682599966254202e-07, 1.0008780009229667e-06, 1.0325433095204062e-06, 1.123110905609792e-06, 4.020138476334978e-06, 3.5560437936510425e-06, 3.115200343017932e-06, 2.6352981876698323e-06, 3.514735453791218e-06, 2.7787118597188964e-06, 1.935526142915478e-06, 1.403948317602044e-06, 1.2939183307025814e-06, 8.860209277372633e-07, 7.869712703723053e-07, 7.174756433414586e-07 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 1.927779726429435e-05, "first_to_last_ratio": 1.245760839846551, "imbalance_abs_log_ratio": 0.21974645960373168, "last_quartile_mean": 1.547471765661612e-05, "mean": 1.932117262981592e-05, "normalized": [ 0.9513832108679868, 1.0804188370793308, 0.9797971700718081, 0.9145548650720269, 1.1425312015279383, 1.059364841855209, 0.9798359577607657, 0.8741541919520497, 0.791922785043975, 0.7413783781025832, 0.7038846163222735, 0.6767848254440803, 0.8786010310302586, 0.8105222350581406, 0.7529822048075973, 0.7101138532674466, 0.8937275707095796, 0.8315301934378441, 0.8340285160151916, 0.9107744775718051, 2.9090130298818027, 2.218912447305513, 1.6752813332436074, 1.2711405397402535, 1.6860861523850705, 1.2447822339063075, 0.8421333542546784, 0.6837796014493468, 0.6374903546184028, 0.4711656375874136, 0.4303886673976222, 0.41153568523209066 ], "peak_layer": 21, "peak_normalized": 2.9090130298818027, "population_cv": 0.5008370479153205, "rest_mean": 1.5914611241088198e-05, "spike_contrast": 2.3699356244134466, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.7716604128945616e-05, "top_five_layers": [ 21, 22, 25, 23, 24 ], "top_five_spike_overlap": 5 }, "values": [ 1.8381839254288934e-05, 2.087495886371471e-05, 1.8930830265162513e-05, 1.7670272427494638e-05, 2.20750425796723e-05, 2.0468170987442136e-05, 1.8931579688796774e-05, 1.6889684047782794e-05, 1.5300876839319244e-05, 1.4324299627332948e-05, 1.359987618343439e-05, 1.3076276445644908e-05, 1.697560219326988e-05, 1.566024002386257e-05, 1.4548499166266993e-05, 1.3720232345804106e-05, 1.7267864677705802e-05, 1.606613841431681e-05, 1.6114408936118707e-05, 1.7597230907995254e-05, 5.620554293273017e-05, 4.287199044483714e-05, 3.236839984310791e-05, 2.455992580507882e-05, 3.257716161897406e-05, 2.4050652427831665e-05, 1.6271003914880566e-05, 1.3211423720349558e-05, 1.2317061191424727e-05, 9.103472621063702e-06, 8.315613740705885e-06, 7.95135201769881e-06 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 1.4843413850940124e-05, "first_to_last_ratio": 1.4030788738423186, "imbalance_abs_log_ratio": 0.33866901753182577, "last_quartile_mean": 1.0579172794678016e-05, "mean": 1.4256177962579386e-05, "normalized": [ 1.0176217861326158, 1.1327153217158388, 1.0326923130981436, 0.9648866803150041, 1.1908934199799213, 1.0960634489995678, 0.9939680738633048, 0.900692377771877, 0.8470138488513046, 0.7955451679739739, 0.7576505388304864, 0.7198047778298761, 0.9366465078267279, 0.8643298223244676, 0.805201793711245, 0.7625376083043307, 0.9767697411936216, 0.9031581137016468, 0.9128386631381071, 0.9814118957842993, 2.8159171308578697, 2.0956146124917954, 1.5087616780297926, 1.0506537160771665, 1.458216451941166, 1.0599886159077407, 0.7470682254259201, 0.6356252549493797, 0.6116181098973799, 0.494790712189984, 0.4631388926042615, 0.4661646982811839 ], "peak_layer": 21, "peak_normalized": 2.8159171308578697, "population_cv": 0.45283030032528887, "rest_mean": 1.2181553689448197e-05, "spike_contrast": 2.089975522542811, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.5459149037487804e-05, "top_five_layers": [ 21, 22, 23, 25, 5 ], "top_five_spike_overlap": 4 }, "values": [ 1.450739728170447e-05, 1.614819120732136e-05, 1.4722245396114886e-05, 1.3755596228293143e-05, 1.697758852969855e-05, 1.5625675587216392e-05, 1.4170185750117525e-05, 1.284043082705466e-05, 1.2075180165993515e-05, 1.1341433491907082e-05, 1.0801200915011577e-05, 1.026166501105763e-05, 1.3352999303606339e-05, 1.2322039765422232e-05, 1.1479100066935644e-05, 1.087087184714619e-05, 1.3925003258918878e-05, 1.2875582797278184e-05, 1.301359043281991e-05, 1.3991182640893385e-05, 4.0144215745385736e-05, 2.9875454856664874e-05, 2.1509174985112622e-05, 1.497830635344144e-05, 2.0788593246834353e-05, 1.511138634668896e-05, 1.0650337571860291e-05, 9.061586752068251e-06, 8.719336619833484e-06, 7.0538244472118095e-06, 6.602590474358294e-06, 6.645726898568682e-06 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 4.959246007274487e-05, "first_to_last_ratio": 0.9907625080220395, "imbalance_abs_log_ratio": 0.009280422189831888, "last_quartile_mean": 5.0054841267410666e-05, "mean": 5.409351444995991e-05, "normalized": [ 0.8395097147540029, 0.9744150134553785, 0.8810986138557345, 0.8293455570233751, 1.0529840630088905, 0.9888544769249366, 0.9441915543103054, 0.8239309095426922, 0.7291828844394362, 0.6845643491425287, 0.6490791770023843, 0.6252182363319296, 0.805904345519456, 0.7388500387185822, 0.6899978948600001, 0.6601185046807971, 0.7971145805539812, 0.7588505605637996, 0.7483060169709823, 0.8370897316424105, 2.891719418727478, 2.339247153891808, 1.8311731018487076, 1.4765417029516001, 1.9445887514487483, 1.4796988518049639, 0.9997439637792762, 0.8002659456053387, 0.7599157216821444, 0.5194334728104613, 0.46331290567462746, 0.43575278647324234 ], "peak_layer": 21, "peak_normalized": 2.891719418727478, "population_cv": 0.5419074124544274, "rest_mean": 4.310798289629424e-05, "spike_contrast": 2.6309601428719196, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00011341538483975456, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 4.541203088592738e-05, 5.27095326106064e-05, 4.7661720600444824e-05, 4.4862215872853994e-05, 5.695960862794891e-05, 5.349061393644661e-05, 5.1074639486614615e-05, 4.4569318561116233e-05, 3.9444064896088094e-05, 3.703049151226878e-05, 3.511097384034656e-05, 3.3820251701399684e-05, 4.359419835964218e-05, 3.996699524577707e-05, 3.7324411096051335e-05, 3.570812987163663e-05, 4.311872908147052e-05, 4.1048893763218075e-05, 4.047850234201178e-05, 4.52811254945118e-05, 0.0001564232661621645, 0.0001265380997210741, 9.905458864523098e-05, 7.98713299445808e-05, 0.00010518963972572237, 8.00421112217009e-05, 5.4079664550954476e-05, 4.328919749241322e-05, 4.1106512071564794e-05, 2.8097982067265548e-05, 2.5062223357963376e-05, 2.357139965170063e-05 ] } } }, "pre_attention_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "ee695e78937de01b7128d3866b8fc5dd328ebdf2c159be37918d4f02952d5248", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.0626930519938469, 0.09030644595623016, 0.20229491591453552, 0.20300470292568207, 0.19049881398677826, 0.12695962190628052, 0.12132706493139267, 0.11585728824138641, 0.13427142798900604, 0.10485932976007462, 0.10419609397649765, 0.10852954536676407, 0.1286265105009079, 0.09096142649650574, 0.10080574452877045, 0.10566592961549759, 0.1287592649459839, 0.09650235623121262, 0.1065765842795372, 0.11446042358875275, 0.12395420670509338, 0.07221997529268265, 0.07677769660949707, 0.08416415005922318, 0.09094838052988052, 0.059439923614263535, 0.062182098627090454, 0.06896103918552399, 0.0776243805885315, 0.06285123527050018, 0.07231374084949493, 0.08016443997621536 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 3.938596876196243e-06, "first_to_last_ratio": 1.4687439500995467, "imbalance_abs_log_ratio": 0.3844075798206889, "last_quartile_mean": 2.6816089189196646e-06, "mean": 4.783229435645353e-06, "normalized": [ 0.7956252786393824, 1.0370472714954742, 0.710498338157694, 0.5964937115446012, 0.3749449264807521, 0.8378820491967393, 1.0684520451305293, 1.1663998119658086, 0.7214030520590701, 0.6071460114163247, 0.48362147796623167, 0.4651818012271395, 0.43991710581847804, 0.7863314511102969, 0.7087273994242284, 0.6417164657893716, 0.8163107747439983, 1.2549364809497514, 1.3705101626081868, 1.5767441185915623, 2.4874551735032497, 3.85688713365935, 2.611503882001361, 2.099245818460862, 1.0160491812931192, 1.053064391512687, 0.6547338043084208, 0.5550746133093791, 0.3848762063898074, 0.2513747113202079, 0.2698186187275936, 0.30002673119834156 ], "peak_layer": 22, "peak_normalized": 3.85688713365935, "population_cv": 0.7731176077888372, "rest_mean": 3.5305297808884426e-06, "spike_contrast": 3.270842702940382, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.1547807571332668e-05, "top_five_layers": [ 22, 23, 21, 24, 20 ], "top_five_spike_overlap": 4 }, "values": [ 3.8056582525314298e-06, 4.96043503517285e-06, 3.398476565052988e-06, 2.8531662792374846e-06, 1.7934476090886164e-06, 4.007782081316691e-06, 5.110651272843825e-06, 5.5791579143260606e-06, 3.450636313573341e-06, 2.904118673541234e-06, 2.3132724891183898e-06, 2.2250712845561793e-06, 2.104224449794856e-06, 3.761203743124497e-06, 3.3900057587743504e-06, 3.069477088502026e-06, 3.904601726389956e-06, 6.002649115544045e-06, 6.555464551638579e-06, 7.541928880527848e-06, 1.1898068805749062e-05, 1.8448376067681238e-05, 1.2491422239691019e-05, 1.0041174391517416e-05, 4.859996352024609e-06, 5.037048595113447e-06, 3.1317420052801026e-06, 2.655049229360884e-06, 1.8409511994832428e-06, 1.2023829185636714e-06, 1.2906043593829963e-06, 1.4350966921483632e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 1.538124422495457e-05, "first_to_last_ratio": 1.4693389111400172, "imbalance_abs_log_ratio": 0.38481257932322294, "last_quartile_mean": 1.0468139180375147e-05, "mean": 1.899974006391858e-05, "normalized": [ 0.7172983669482088, 1.0339221692013465, 0.6878312449702983, 0.61914282410142, 0.38449687390128023, 0.8396762391515767, 1.009924775322266, 1.1841094669395977, 0.6995162576825456, 0.6148703903552607, 0.5112097922373936, 0.47816825931240625, 0.4500638662498898, 0.8512672854151476, 0.7022850846434918, 0.645592153504749, 0.8318146508688388, 1.3054875993158614, 1.298759927098455, 1.5837061220619262, 2.5001411261892037, 3.8847405290702386, 2.6775582061984458, 2.0807189385912803, 1.0049130080051407, 1.059855281496919, 0.6540585161698493, 0.4911735413170337, 0.38485962364558457, 0.25176749400043913, 0.2642039271549161, 0.296866458878989 ], "peak_layer": 22, "peak_normalized": 3.8847405290702386, "population_cv": 0.780882099528633, "rest_mean": 1.3969684274612788e-05, "spike_contrast": 3.3044441390891333, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.616204132616986e-05, "top_five_layers": [ 22, 23, 21, 24, 20 ], "top_five_spike_overlap": 4 }, "values": [ 1.3628482520289253e-05, 1.9644252461148426e-05, 1.3068614862277173e-05, 1.1763552720367443e-05, 7.305340659513604e-06, 1.5953630281728692e-05, 1.918830821523443e-05, 2.249777207907755e-05, 1.3290627066453453e-05, 1.1682377589750104e-05, 9.712853170640301e-06, 9.085072633752134e-06, 8.551096470910124e-06, 1.6173857147805393e-05, 1.3343234058993403e-05, 1.2266083103895653e-05, 1.580426214786712e-05, 2.480392504367046e-05, 2.4676101020304486e-05, 3.0090004656813107e-05, 4.750203152070753e-05, 7.380906026810408e-05, 5.087290992378257e-05, 3.953311897930689e-05, 1.9093085938948207e-05, 2.0136974853812717e-05, 1.2426941793819424e-05, 9.332169611298013e-06, 7.312232810363639e-06, 4.783516942552524e-06, 5.019805939809885e-06, 5.640385552396765e-06 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.013640241581015289, "first_to_last_ratio": 1.4693389283077047, "imbalance_abs_log_ratio": 0.38481259100717635, "last_quartile_mean": 0.009283250663429499, "mean": 0.016849159568664618, "normalized": [ 0.7172983623125343, 1.033922210661322, 0.6878312280968879, 0.6191428451397352, 0.38449688266839166, 0.8396762176755517, 1.0099248381652821, 1.1841095291912374, 0.699516233677964, 0.6148703762706434, 0.5112097772115105, 0.478168231068241, 0.45006385280028943, 0.8512673125159866, 0.7022850804883993, 0.645592177879939, 0.8318146892353585, 1.3054876414312566, 1.2987598958704207, 1.5837060871432236, 2.5001411524136627, 3.8847403293530753, 2.6775582743476143, 2.0807188708282953, 1.0049130225823506, 1.059855283061569, 0.6540585158104759, 0.49117356970505077, 0.38485961912245675, 0.2517674823850363, 0.26420393971361933, 0.29686647117261844 ], "peak_layer": 22, "peak_normalized": 3.8847403293530753, "population_cv": 0.7808820795681298, "rest_mean": 0.012388455871216677, "spike_contrast": 3.304444069579033, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0409369595348835, "top_five_layers": [ 22, 23, 21, 24, 20 ], "top_five_spike_overlap": 4 }, "values": [ 0.012085874564945698, 0.01742072030901909, 0.011589378118515015, 0.010432036593556404, 0.006478449329733849, 0.014147838577628136, 0.01701638475060463, 0.01995125040411949, 0.011786260642111301, 0.010360049083828926, 0.00861345510929823, 0.008056732825934887, 0.007583197671920061, 0.01434313878417015, 0.011832913383841515, 0.010877685621380806, 0.014015378430485725, 0.02199636958539486, 0.02188301272690296, 0.026684116572141647, 0.04212527722120285, 0.06545460969209671, 0.045114606618881226, 0.035058364272117615, 0.016931939870119095, 0.017857670783996582, 0.011020336300134659, 0.008275861851871014, 0.006484561134129763, 0.004242070484906435, 0.004451614338904619, 0.005001950543373823 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 1.116900129716214e-06, "first_to_last_ratio": 0.5683712228091384, "imbalance_abs_log_ratio": 0.5649805124554581, "last_quartile_mean": 1.9650891616151966e-06, "mean": 1.529344047845882e-06, "normalized": [ 2.6880209079483035, 1.0614582941932853, 0.2102909530100704, 0.26575956316042637, 0.20909128403339697, 0.35378989483255807, 0.45306549936231316, 0.601029120982115, 0.4546779430766579, 0.4200925455954701, 0.3563560449981123, 0.3858330043229325, 0.31101090672883425, 0.5478281050493286, 0.5621266026346432, 0.6679908063561342, 0.4195538997444655, 0.7825840314762202, 0.6580446653717678, 0.8744406926028508, 1.5303961911244068, 2.9172827358993394, 2.706267368892852, 2.283625882145698, 1.5407128276816973, 2.117876986285686, 1.7468304275698725, 1.626947456367584, 1.1368682673119774, 0.9035383474210811, 0.619607831116708, 0.5870009127032113 ], "peak_layer": 22, "peak_normalized": 2.9172827358993394, "population_cv": 0.7864680067735266, "rest_mean": 1.1907198037769611e-06, "spike_contrast": 2.820071485471858, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.357914965818054e-06, "top_five_layers": [ 22, 23, 1, 24, 26 ], "top_five_spike_overlap": 3 }, "values": [ 4.1109087760560215e-06, 1.623334924261144e-06, 3.216072173017892e-07, 4.064378060775198e-07, 3.1977251069292834e-07, 5.410664698501932e-07, 6.928930247340759e-07, 9.191803087560402e-07, 6.953590059310955e-07, 6.42466034150857e-07, 5.449909963317623e-07, 5.900714086237713e-07, 4.756426790208934e-07, 8.378176516998792e-07, 8.596849738751189e-07, 1.021587763716525e-06, 6.416422593247262e-07, 1.1968402304773917e-06, 1.0063766922030482e-06, 1.3373206684264005e-06, 2.3405023057421204e-06, 4.461528988031205e-06, 4.1388138924958184e-06, 3.492449650366325e-06, 2.3562799924548017e-06, 3.2389625630457886e-06, 2.6715047170000616e-06, 2.4881624085537624e-06, 1.7386627177984337e-06, 1.381820993628935e-06, 9.475935485170339e-07, 8.977263519227563e-07 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 1.0329262465802458e-05, "first_to_last_ratio": 1.4941372208967194, "imbalance_abs_log_ratio": 0.40154893048262924, "last_quartile_mean": 6.91319533530077e-06, "mean": 1.2918823138363678e-05, "normalized": [ 0.5650535478116867, 1.028589581339238, 0.7051164877557733, 0.646035483201396, 0.4372682856255746, 0.9021782951846146, 1.0782246370785644, 1.0339444727840728, 0.750096224198186, 0.6641971843619161, 0.5460080005677336, 0.49482360055494723, 0.5015633097451457, 0.9124642656227907, 0.743882514132793, 0.6507004473996142, 0.8420053751613578, 1.300495682091372, 1.3063226104942232, 1.5633845745317094, 2.3203289602592174, 3.884609736204134, 2.789783055946839, 2.0519174114119347, 0.9845286449473958, 1.1046340664686856, 0.6339813277911774, 0.42233072712967584, 0.4081137906027939, 0.2584902894398306, 0.2181840356851096, 0.2507433744704966 ], "peak_layer": 22, "peak_normalized": 3.884609736204134, "population_cv": 0.7759716411307664, "rest_mean": 9.55458560585814e-06, "spike_contrast": 3.2534855090768366, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.108570581389358e-05, "top_five_layers": [ 22, 23, 21, 24, 20 ], "top_five_spike_overlap": 4 }, "values": [ 7.2998268478841055e-06, 1.3288166883285157e-05, 9.109275197261013e-06, 8.346018148586154e-06, 5.648991646012291e-06, 1.1655081834760495e-05, 1.3929393389844336e-05, 1.3357345778786112e-05, 9.690360457170755e-06, 8.580645953770727e-06, 7.053780791466124e-06, 6.392538580257678e-06, 6.479607691289857e-06, 1.178796446765773e-05, 9.610086635802872e-06, 8.406283996009734e-06, 1.0877718523261137e-05, 1.680087370914407e-05, 1.6876150766620412e-05, 2.01970888156211e-05, 2.9975919460412115e-05, 5.0184586143586785e-05, 3.6040713894180954e-05, 2.6508358132559806e-05, 1.2718951438728254e-05, 1.4270572137320414e-05, 8.19029264675919e-06, 5.456015969684813e-06, 5.272349881124683e-06, 3.339390332257608e-06, 2.8186809686303604e-06, 3.2393093079008395e-06 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 6.8430363171501085e-06, "first_to_last_ratio": 1.5026996054769635, "imbalance_abs_log_ratio": 0.4072632275067286, "last_quartile_mean": 4.553828517828151e-06, "mean": 8.620025433003775e-06, "normalized": [ 0.47616845005868963, 1.0148611000743992, 0.710223794261972, 0.6672402729910655, 0.4961589158067777, 0.9546874410900915, 1.1397903797461284, 0.8916966780582295, 0.8004354318330394, 0.7285742201106149, 0.5845932983542859, 0.5137663436515674, 0.5545480923657561, 0.9813050649779153, 0.7876391323577042, 0.6647739880805192, 0.8418654837614679, 1.3131945289007125, 1.3155087746003027, 1.5382584577546115, 2.139613947229906, 3.8273584966992447, 2.8688986684137876, 1.9625605404838322, 0.9339731800698603, 1.1527216258910076, 0.6282997180664591, 0.3737066830161134, 0.44061949849021204, 0.2741895282698357, 0.20211065041819368, 0.2206576141156973 ], "peak_layer": 22, "peak_normalized": 3.8273584966992447, "population_cv": 0.7603736250497635, "rest_mean": 6.470636511342794e-06, "spike_contrast": 3.1259251813196967, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.0226725609973073e-05, "top_five_layers": [ 22, 23, 21, 24, 20 ], "top_five_spike_overlap": 4 }, "values": [ 4.1045841498998925e-06, 8.74812849360751e-06, 6.1221471696626395e-06, 5.751628123107366e-06, 4.276902473066002e-06, 8.229430022765882e-06, 9.825022061704658e-06, 7.686448043386918e-06, 6.89977377987816e-06, 6.280328307184391e-06, 5.0392090997775085e-06, 4.428678948897868e-06, 4.780218660016544e-06, 8.458874617645051e-06, 6.7894693529524375e-06, 5.730368684453424e-06, 7.2569018811918795e-06, 1.1319770237605553e-05, 1.133971909439424e-05, 1.3259827028377913e-05, 1.8443526641931385e-05, 3.2991927582770586e-05, 2.4729979486437514e-05, 1.691732177278027e-05, 8.05087256594561e-06, 9.936489732353948e-06, 5.415959549281979e-06, 3.221361112082377e-06, 3.798151283262996e-06, 2.3635207071492914e-06, 1.7421989468857646e-06, 1.9020742456632433e-06 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 3.044255652184802e-05, "first_to_last_ratio": 1.5007576827872244, "imbalance_abs_log_ratio": 0.40597010243521225, "last_quartile_mean": 2.0284791389713064e-05, "mean": 3.804233946880231e-05, "normalized": [ 0.5857759674584327, 1.0498401471669287, 0.6988735722072594, 0.6414514551938159, 0.38168573834485325, 0.8650779544044107, 1.0390116098583848, 1.1401099302792224, 0.6957523133167004, 0.5943117340775991, 0.5056541248372528, 0.47166554819599, 0.443309132558095, 0.8367000220763482, 0.6967329006099322, 0.6307446067218856, 0.8295035976978059, 1.2829704249798906, 1.2892643915579425, 1.5965252135223877, 2.5505102273284863, 4.067387347932314, 2.6645940137099884, 2.176818489251816, 1.0232686605155923, 1.0772178881062735, 0.6429491602740723, 0.44273014233312213, 0.3730667260421724, 0.22770687651908533, 0.21960584267103583, 0.2591842402509037 ], "peak_layer": 22, "peak_normalized": 4.067387347932314, "population_cv": 0.8152788912728794, "rest_mean": 2.749956908061264e-05, "spike_contrast": 3.45362864729336, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.497329956502654e-05, "top_five_layers": [ 22, 23, 21, 24, 20 ], "top_five_spike_overlap": 4 }, "values": [ 2.2284288206719793e-05, 3.993837526650168e-05, 2.658678567968309e-05, 2.4402314011240378e-05, 1.4520218428515363e-05, 3.290958920842968e-05, 3.952643237425946e-05, 4.337244899943471e-05, 2.6468045689398423e-05, 2.260900873807259e-05, 1.9236265870858915e-05, 1.794326090021059e-05, 1.6864516510395333e-05, 3.183002627338283e-05, 2.650534952408634e-05, 2.399500044703018e-05, 3.1556257454212755e-05, 4.880719643551856e-05, 4.904663364868611e-05, 6.073555414332077e-05, 9.702737588668242e-05, 0.00015473293024115264, 0.00010136739001609385, 8.281126793008298e-05, 3.892753375112079e-05, 4.097988858120516e-05, 2.4459290216327645e-05, 1.6842490367707796e-05, 1.4192331036610994e-05, 8.662502295919694e-06, 8.354320016223937e-06, 9.859974852588493e-06 ] } } }, "pre_mlp_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "e53fa425d89dfeb0e9395eca9bcd73dc0a55314abf45c5b10b43e2cbfa724345", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.059514161199331284, 0.08157797157764435, 0.14292451739311218, 0.10997853428125381, 0.0991024449467659, 0.1065012738108635, 0.12155740708112717, 0.1095810979604721, 0.06964563578367233, 0.08311934769153595, 0.09363401681184769, 0.0941167026758194, 0.08564268052577972, 0.09493856877088547, 0.09063401818275452, 0.09473779052495956, 0.10505300015211105, 0.09355910867452621, 0.09618847072124481, 0.11416846513748169, 0.047205954790115356, 0.053352247923612595, 0.06512638926506042, 0.07829347252845764, 0.05368548259139061, 0.05310077592730522, 0.061235878616571426, 0.07320034503936768, 0.05304349213838577, 0.0639304369688034, 0.07404901832342148, 0.08157160878181458 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.8519853952625454e-06, "first_to_last_ratio": 0.18647050298551093, "imbalance_abs_log_ratio": 1.6794822133426248, "last_quartile_mean": 9.931787417372107e-06, "mean": 5.88188727768113e-06, "normalized": [ 0.6908731928777746, 0.2510898081805258, 0.1555018826120805, 0.30498887791309987, 0.23852506507874588, 0.1850969485210883, 0.24724001820980046, 0.44558374728991557, 0.5188458231301253, 0.39249267749677386, 0.36080177718358364, 0.3701615414805411, 0.4386910849035176, 0.4549369013108399, 0.5189021844310387, 0.5324765988761366, 0.423744941540098, 0.538218841293273, 0.4983028247657117, 0.5711917099344411, 2.628693857876261, 2.954575363636704, 2.561389805510594, 2.2093742835290766, 2.488094137098029, 2.5215495558896106, 2.1153495626085475, 1.390737907423905, 1.7468037117541915, 1.4229552331114703, 0.9251937165053545, 0.8976164180271445 ], "peak_layer": 22, "peak_normalized": 2.954575363636704, "population_cv": 0.8744988545997188, "rest_mean": 4.173498030855559e-06, "spike_contrast": 3.619790664533337, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.5107189210539217e-05, "top_five_layers": [ 22, 21, 23, 26, 25 ], "top_five_spike_overlap": 4 }, "values": [ 4.063638243678724e-06, 1.47688194829243e-06, 9.146445449914609e-07, 1.7939102008313057e-06, 1.402977545694739e-06, 1.0887193866437883e-06, 1.4542379176418763e-06, 2.620873374326038e-06, 3.051792646147078e-06, 2.308597686351277e-06, 2.1221953829808626e-06, 2.1772484615212306e-06, 2.5803315111261327e-06, 2.6758875719679054e-06, 3.052124156965874e-06, 3.131967332592467e-06, 2.492419980626437e-06, 3.1657425552111818e-06, 2.930961045422009e-06, 3.35968525178032e-06, 1.546168095956091e-05, 1.737847924232483e-05, 1.5065806110214908e-05, 1.2995290489925537e-05, 1.4634689250669908e-05, 1.4831470252829604e-05, 1.244224768015556e-05, 8.180163604265545e-06, 1.0274502528773155e-05, 8.369662282348145e-06, 5.441885150503367e-06, 5.279678589431569e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 7.456590537913144e-06, "first_to_last_ratio": 0.1876899442113098, "imbalance_abs_log_ratio": 1.6729639105525023, "last_quartile_mean": 3.972823674303072e-05, "mean": 2.3512681963211435e-05, "normalized": [ 0.7107474828040198, 0.25633599856936284, 0.15597120637969672, 0.30725052014529014, 0.23594972380724819, 0.18562628701524886, 0.24597569145694365, 0.439187753490902, 0.5217731787787261, 0.3934518092193065, 0.35809994155163755, 0.37309394567202037, 0.4386907796473115, 0.4532813019679985, 0.5205465647100695, 0.5270539126771521, 0.42492300278530837, 0.5404789756339394, 0.5029417459456188, 0.5683710910710696, 2.625962443954409, 2.9406818063717424, 2.5595107592674626, 2.1968826232108496, 2.498494532589649, 2.5209654246358313, 2.119993559463838, 1.3957420657255426, 1.7454055590978041, 1.4182097085654002, 0.9276855395297009, 0.8907150642588996 ], "peak_layer": 22, "peak_normalized": 2.9406818063717424, "population_cv": 0.8731331337518368, "rest_mean": 1.670137832358402e-05, "spike_contrast": 3.6101045344298743, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.029372161719948e-05, "top_five_layers": [ 22, 21, 23, 26, 25 ], "top_five_spike_overlap": 4 }, "values": [ 1.6711579519324005e-05, 6.02714681008365e-06, 3.667301371024223e-06, 7.224283763207495e-06, 5.547810815187404e-06, 4.364571850601351e-06, 5.78354820390814e-06, 1.0326481969968881e-05, 1.2268286809558049e-05, 9.251107258023694e-06, 8.419890036748257e-06, 8.7724392869859e-06, 1.0314796782040503e-05, 1.0657859093043953e-05, 1.2239445823070128e-05, 1.2392451026244089e-05, 9.991079423343763e-06, 1.2708110261883121e-05, 1.1825509318441618e-05, 1.3363928701437544e-05, 6.174341979203746e-05, 6.914331606822088e-05, 6.018096246407367e-05, 5.165460243006237e-05, 5.874630733160302e-05, 5.9274658269714564e-05, 4.984673432772979e-05, 3.2817639294080436e-05, 4.103916580788791e-05, 3.334591383463703e-05, 2.1812375052832067e-05, 2.094310002576094e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.006612578930798918, "first_to_last_ratio": 0.18768994142786996, "imbalance_abs_log_ratio": 1.6729639253824915, "last_quartile_mean": 0.03523139748722315, "mean": 0.02085128170438111, "normalized": [ 0.7107474595690433, 0.25633598480800296, 0.15597120907260567, 0.30725051583333624, 0.23594971582617197, 0.185626292626144, 0.24597569154071516, 0.43918772090263997, 0.5217731787613039, 0.3934518290670755, 0.358099947615767, 0.37309396702760195, 0.43869077807139933, 0.4532813161722498, 0.5205465437645069, 0.5270539221495462, 0.42492301400382976, 0.5404789716436743, 0.5029417432235066, 0.5683711051684076, 2.6259625852653525, 2.9406818490235107, 2.559510707390594, 2.1968826886965482, 2.4984945604669404, 2.5209655238317774, 2.11999349275561, 1.3957419713008816, 1.7454054366890155, 1.4182097187854519, 0.9276855165674615, 0.890715042379328 ], "peak_layer": 22, "peak_normalized": 2.9406818490235107, "population_cv": 0.8731331446714019, "rest_mean": 0.014810949288032673, "spike_contrast": 3.610104640346447, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.05346907675266266, "top_five_layers": [ 22, 21, 23, 26, 25 ], "top_five_spike_overlap": 4 }, "values": [ 0.014819995500147343, 0.005344933830201626, 0.0032521996181458235, 0.006406567059457302, 0.004919853992760181, 0.00387054611928761, 0.005128908436745405, 0.009157626889646053, 0.01087963953614235, 0.008203974924981594, 0.007466842886060476, 0.007779487408697605, 0.009147264994680882, 0.009451496414840221, 0.010854062624275684, 0.010989749804139137, 0.008860189467668533, 0.011269679293036461, 0.010486979968845844, 0.011851266026496887, 0.05475468561053276, 0.06131698563694954, 0.053369078785181046, 0.045807819813489914, 0.052096813917160034, 0.05256536230444908, 0.044204581528902054, 0.029103009030222893, 0.036393940448760986, 0.029571490362286568, 0.019343432039022446, 0.018572550266981125 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 4.307105996304017e-07, "first_to_last_ratio": 0.15820955550394056, "imbalance_abs_log_ratio": 1.843834824057122, "last_quartile_mean": 2.722405724853161e-06, "mean": 1.5342757890834946e-06, "normalized": [ 0.7081018956127941, 0.212942667688312, 0.14407277621078163, 0.32033103894754916, 0.20958258065348837, 0.16001428536225384, 0.1928849750424583, 0.2978751445241861, 0.43614209802624065, 0.35456800397013627, 0.3205389951131459, 0.3265340457063885, 0.39765616581116076, 0.41829943678862663, 0.5105501700709926, 0.4933210036037538, 0.43054180477210957, 0.528664659989701, 0.5051109289194535, 0.414024312985959, 2.59604933949905, 2.9264634923258344, 2.666716645798573, 2.2338825438809438, 2.546646393997065, 2.602172802005701, 2.346920001957485, 1.4784227347327796, 1.7831647304083158, 1.4806483436370603, 0.9963666347410605, 0.9607893472166396 ], "peak_layer": 22, "peak_normalized": 2.9264634923258344, "population_cv": 0.9145189641614374, "rest_mean": 1.0813940342039267e-06, "spike_contrast": 3.680284095854974, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.979837265433161e-06, "top_five_layers": [ 22, 23, 26, 21, 25 ], "top_five_spike_overlap": 4 }, "values": [ 1.086423594642838e-06, 3.2671277949702926e-07, 2.2104737240624672e-07, 4.914761575491866e-07, 3.21557479310286e-07, 2.455060439388035e-07, 2.9593874728561786e-07, 4.570226224132057e-07, 6.691622616017412e-07, 5.440051040750404e-07, 4.917952196592523e-07, 5.009932806387951e-07, 6.101142275838356e-07, 6.417866984520515e-07, 7.833247650523845e-07, 7.568904720756109e-07, 6.605698672501603e-07, 8.111173883662559e-07, 7.749794690425915e-07, 6.35227479506284e-07, 3.98305564885959e-06, 4.490002083912259e-06, 4.0914787859946955e-06, 3.4273919027327793e-06, 3.9072579056664836e-06, 3.992450729128905e-06, 3.600822537919157e-06, 2.2683082079311134e-06, 2.7358664738130756e-06, 2.27172290578892e-06, 1.5287012047338067e-06, 1.4741158338438254e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 5.223859773195727e-06, "first_to_last_ratio": 0.20962609729315076, "imbalance_abs_log_ratio": 1.5624298242995687, "last_quartile_mean": 2.4919892325669935e-05, "mean": 1.5486922322338614e-05, "normalized": [ 0.7715312731558687, 0.27317941161013015, 0.13552825139565777, 0.2927753710438871, 0.2618690157194892, 0.20648596040205808, 0.2641975851840613, 0.4928955706841361, 0.5886841012409418, 0.44043692518303496, 0.40542502076823556, 0.42857470454714536, 0.506842146108589, 0.5154291534251697, 0.5826795973216162, 0.5933989097172402, 0.4769966579273341, 0.6178877367586311, 0.5669062127288081, 0.6155582863946625, 2.590918687915948, 2.900503381242867, 2.526846529041888, 2.0717085968279685, 2.5063565741739886, 2.3330216475750456, 1.9501745481507984, 1.31308531870814, 1.6771377835162664, 1.3577552572890352, 0.8929771731667675, 0.8422326110745892 ], "peak_layer": 22, "peak_normalized": 2.900503381242867, "population_cv": 0.8259135305419505, "rest_mean": 1.112974339573864e-05, "spike_contrast": 3.505533518492153, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.901568852597848e-05, "top_five_layers": [ 22, 21, 23, 25, 26 ], "top_five_spike_overlap": 4 }, "values": [ 1.1948644896619953e-05, 4.230708327668253e-06, 2.098915501846932e-06, 4.534189429250546e-06, 4.055545105074998e-06, 3.1978320294001605e-06, 4.091607479494996e-06, 7.633435416209977e-06, 9.116904948314186e-06, 6.821012448199326e-06, 6.278785804170184e-06, 6.6373031586408615e-06, 7.849424946471117e-06, 7.982411261764355e-06, 9.023913662531413e-06, 9.189922820951324e-06, 7.387210189335747e-06, 9.56917938310653e-06, 8.779632480582222e-06, 9.533103366266005e-06, 4.012535646324977e-05, 4.4919870560988784e-05, 3.9133075915742666e-05, 3.208439011359587e-05, 3.8815749576315284e-05, 3.613132503232919e-05, 3.020220174221322e-05, 2.0335650333436206e-05, 2.5973702577175573e-05, 2.102745020238217e-05, 1.3829468116455246e-05, 1.3043591025052592e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 3.515129350262214e-06, "first_to_last_ratio": 0.22789268493442033, "imbalance_abs_log_ratio": 1.4788804409445422, "last_quartile_mean": 1.5424493994942168e-05, "mean": 1.007265868935292e-05, "normalized": [ 0.8015519377376712, 0.2805852368786066, 0.1215369164413179, 0.2645493981261726, 0.28294934408838834, 0.22357447435911756, 0.2767016495904134, 0.5403695357271925, 0.6481311540844804, 0.468018284006402, 0.4310931551221704, 0.4771032909031079, 0.5769491568777, 0.5843394016136497, 0.6535843342936872, 0.6750886128416339, 0.5350348656187338, 0.7087273324786804, 0.638961948582862, 0.6568327300152229, 2.6106484901987757, 2.9055624786185823, 2.497862991650572, 1.889659222335943, 2.462812353959888, 2.1421768429802923, 1.7611574334136273, 1.2455853123406186, 1.5573178930849627, 1.3669392943330865, 0.898827195473597, 0.8157677322228438 ], "peak_layer": 22, "peak_normalized": 2.9055624786185823, "population_cv": 0.7843234888656763, "rest_mean": 7.324484655597459e-06, "spike_contrast": 3.4013039337305124, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 2.4912798471632415e-05, "top_five_layers": [ 22, 21, 23, 25, 26 ], "top_five_spike_overlap": 4 }, "values": [ 8.073759090621024e-06, 2.8262393243494444e-06, 1.2241998774698004e-06, 2.6647157937986776e-06, 2.850052169378614e-06, 2.2519893718708772e-06, 2.7871212751051644e-06, 5.442957899504108e-06, 6.528403901029378e-06, 4.714188435173128e-06, 4.342254214861896e-06, 4.805698608834064e-06, 5.811411938339006e-06, 5.885851351195015e-06, 6.583331924048252e-06, 6.799937182222493e-06, 5.389223588281311e-06, 7.138768523873296e-06, 6.436045623559039e-06, 6.616051905439235e-06, 2.629617119964678e-05, 2.926673914771527e-05, 2.5160121367662214e-05, 1.9033892385778017e-05, 2.4807068257359788e-05, 2.157741619157605e-05, 1.773953772499226e-05, 1.2546355719678104e-05, 1.5686331607867032e-05, 1.3768712960882112e-05, 9.053579560713843e-06, 8.216949936468154e-06 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 1.5244429164340545e-05, "first_to_last_ratio": 0.17914503168248408, "imbalance_abs_log_ratio": 1.7195595683397966, "last_quartile_mean": 8.509546159984893e-05, "mean": 4.8921966225634606e-05, "normalized": [ 0.7169042930017834, 0.2524583501541368, 0.13310336431350794, 0.29880827490234857, 0.2343888184895342, 0.1809389294746084, 0.244661679968318, 0.4315926224165226, 0.5130992581225174, 0.4067752347707178, 0.36711717074732164, 0.37451389871845103, 0.42008942691793005, 0.42983788129440553, 0.5151278777695638, 0.5230352185268532, 0.4281426676659637, 0.5420913412476243, 0.4983657386698305, 0.557394629165695, 2.4426745997803625, 2.7520031043160365, 2.5674501755548946, 2.2541283838107753, 2.5284521937271367, 2.4856666124454687, 2.2081356797967824, 1.4561106117039475, 1.8439362857756765, 1.4840893517231049, 0.9710586561854326, 0.9378476688427477 ], "peak_layer": 22, "peak_normalized": 2.7520031043160365, "population_cv": 0.8635789779744573, "rest_mean": 3.5251522806194975e-05, "spike_contrast": 3.4819023667550137, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012274236069060863, "top_five_layers": [ 22, 23, 25, 26, 21 ], "top_five_spike_overlap": 4 }, "values": [ 3.50723676092457e-05, 1.2350758879620116e-05, 6.511678293463774e-06, 1.4618288332712837e-05, 1.1466761861811392e-05, 8.851888196659274e-06, 1.1969330444117077e-05, 2.1114359697094187e-05, 2.5101824576267973e-05, 1.9900244296877645e-05, 1.7960093828151003e-05, 1.83219563041348e-05, 2.055160075542517e-05, 2.1028514311183244e-05, 2.5201068638125435e-05, 2.5587911295588128e-05, 2.0945581127307378e-05, 2.652017428772524e-05, 2.438103183521889e-05, 2.7268841222394258e-05, 0.00011950044427067041, 0.00013463340292219073, 0.0001256047107744962, 0.00011027639266103506, 0.0001236968528246507, 0.0001216036980622448, 0.00010802633914863691, 7.123579416656867e-05, 9.020898869493976e-05, 7.26045691408217e-05, 4.750609878101386e-05, 4.588135197991505e-05 ] } } } }, "selector": { "expected_uniform_count": 1, "identities_unique": true, "identity_order_sha256": "97fd9721a263024f6a746638f32e0ad9a988cd4c111c00e9e178c1022e5c9d5e", "passed": true, "uniform_count": 1, "uniform_indices": [ 64 ], "visit_count": 65, "visits": [ { "branch": "attention", "coefficient": "detached_learned", "group": 1, "index": 0, "kind": "depth", "layer": 1, "offset": 1, "sources": 1, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 1, "index": 1, "kind": "depth", "layer": 1, "offset": 1, "sources": 2, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 1, "index": 2, "kind": "depth", "layer": 2, "offset": 2, "sources": 2, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 1, "index": 3, "kind": "depth", "layer": 2, "offset": 2, "sources": 2, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 1, "index": 4, "kind": "depth", "layer": 3, "offset": 3, "sources": 2, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 1, "index": 5, "kind": "depth", "layer": 3, "offset": 3, "sources": 2, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 1, "index": 6, "kind": "depth", "layer": 4, "offset": 4, "sources": 2, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 1, "index": 7, "kind": "depth", "layer": 4, "offset": 4, "sources": 2, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 2, "index": 8, "kind": "depth", "layer": 5, "offset": 1, "sources": 2, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 2, "index": 9, "kind": "depth", "layer": 5, "offset": 1, "sources": 3, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 2, "index": 10, "kind": "depth", "layer": 6, "offset": 2, "sources": 3, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 2, "index": 11, "kind": "depth", "layer": 6, "offset": 2, "sources": 3, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 2, "index": 12, "kind": "depth", "layer": 7, "offset": 3, "sources": 3, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 2, "index": 13, "kind": "depth", "layer": 7, "offset": 3, "sources": 3, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 2, "index": 14, "kind": "depth", "layer": 8, "offset": 4, "sources": 3, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 2, "index": 15, "kind": "depth", "layer": 8, "offset": 4, "sources": 3, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 3, "index": 16, "kind": "depth", "layer": 9, "offset": 1, "sources": 3, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 3, "index": 17, "kind": "depth", "layer": 9, "offset": 1, "sources": 4, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 3, "index": 18, "kind": "depth", "layer": 10, "offset": 2, "sources": 4, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 3, "index": 19, "kind": "depth", "layer": 10, "offset": 2, "sources": 4, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 3, "index": 20, "kind": "depth", "layer": 11, "offset": 3, "sources": 4, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 3, "index": 21, "kind": "depth", "layer": 11, "offset": 3, "sources": 4, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 3, "index": 22, "kind": "depth", "layer": 12, "offset": 4, "sources": 4, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 3, "index": 23, "kind": "depth", "layer": 12, "offset": 4, "sources": 4, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 4, "index": 24, "kind": "depth", "layer": 13, "offset": 1, "sources": 4, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 4, "index": 25, "kind": "depth", "layer": 13, "offset": 1, "sources": 5, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 4, "index": 26, "kind": "depth", "layer": 14, "offset": 2, "sources": 5, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 4, "index": 27, "kind": "depth", "layer": 14, "offset": 2, "sources": 5, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 4, "index": 28, "kind": "depth", "layer": 15, "offset": 3, "sources": 5, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 4, "index": 29, "kind": "depth", "layer": 15, "offset": 3, "sources": 5, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 4, "index": 30, "kind": "depth", "layer": 16, "offset": 4, "sources": 5, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 4, "index": 31, "kind": "depth", "layer": 16, "offset": 4, "sources": 5, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 5, "index": 32, "kind": "depth", "layer": 17, "offset": 1, "sources": 5, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 5, "index": 33, "kind": "depth", "layer": 17, "offset": 1, "sources": 6, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 5, "index": 34, "kind": "depth", "layer": 18, "offset": 2, "sources": 6, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 5, "index": 35, "kind": "depth", "layer": 18, "offset": 2, "sources": 6, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 5, "index": 36, "kind": "depth", "layer": 19, "offset": 3, "sources": 6, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 5, "index": 37, "kind": "depth", "layer": 19, "offset": 3, "sources": 6, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 5, "index": 38, "kind": "depth", "layer": 20, "offset": 4, "sources": 6, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 5, "index": 39, "kind": "depth", "layer": 20, "offset": 4, "sources": 6, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 6, "index": 40, "kind": "depth", "layer": 21, "offset": 1, "sources": 6, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 6, "index": 41, "kind": "depth", "layer": 21, "offset": 1, "sources": 7, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 6, "index": 42, "kind": "depth", "layer": 22, "offset": 2, "sources": 7, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 6, "index": 43, "kind": "depth", "layer": 22, "offset": 2, "sources": 7, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 6, "index": 44, "kind": "depth", "layer": 23, "offset": 3, "sources": 7, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 6, "index": 45, "kind": "depth", "layer": 23, "offset": 3, "sources": 7, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 6, "index": 46, "kind": "depth", "layer": 24, "offset": 4, "sources": 7, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 6, "index": 47, "kind": "depth", "layer": 24, "offset": 4, "sources": 7, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 7, "index": 48, "kind": "depth", "layer": 25, "offset": 1, "sources": 7, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 7, "index": 49, "kind": "depth", "layer": 25, "offset": 1, "sources": 8, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 7, "index": 50, "kind": "depth", "layer": 26, "offset": 2, "sources": 8, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 7, "index": 51, "kind": "depth", "layer": 26, "offset": 2, "sources": 8, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 7, "index": 52, "kind": "depth", "layer": 27, "offset": 3, "sources": 8, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 7, "index": 53, "kind": "depth", "layer": 27, "offset": 3, "sources": 8, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 7, "index": 54, "kind": "depth", "layer": 28, "offset": 4, "sources": 8, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 7, "index": 55, "kind": "depth", "layer": 28, "offset": 4, "sources": 8, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 8, "index": 56, "kind": "depth", "layer": 29, "offset": 1, "sources": 8, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 8, "index": 57, "kind": "depth", "layer": 29, "offset": 1, "sources": 9, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 8, "index": 58, "kind": "depth", "layer": 30, "offset": 2, "sources": 9, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 8, "index": 59, "kind": "depth", "layer": 30, "offset": 2, "sources": 9, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 8, "index": 60, "kind": "depth", "layer": 31, "offset": 3, "sources": 9, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 8, "index": 61, "kind": "depth", "layer": 31, "offset": 3, "sources": 9, "use_uniform": false }, { "branch": "attention", "coefficient": "detached_learned", "group": 8, "index": 62, "kind": "depth", "layer": 32, "offset": 4, "sources": 9, "use_uniform": false }, { "branch": "mlp", "coefficient": "detached_learned", "group": 8, "index": 63, "kind": "depth", "layer": 32, "offset": 4, "sources": 9, "use_uniform": false }, { "branch": "output", "coefficient": "uniform", "group": null, "index": 64, "kind": "output", "layer": null, "offset": null, "sources": 9, "use_uniform": true } ], "visits_sha256": "260d93ecd556b53e98ae4b3aff8f489a0abe28f33f7d265e812410ecff18ed77" } } }, "parent_learned": { "bits_per_byte": 1.8740129107275643, "forward": { "activation_sha256": { "attention_branch_output": "5e6ecbcd78543fc42d4098a51169dca446d404ce384015643e84d2c78433c949", "mlp_branch_output": "c0b8018c349b1bd09e5d17858eb312a2323da0e92de9651e2bb08ae4adf115c7", "post_attention_state": "90a633f12d984935028fe5d7212be48e7da7b4a0806751939a551938d16dc540", "post_mlp_state": "89f2226456bfe96e245d2aad0f699d32c676f9f53099789cd870253d002402f5", "pre_attention_input": "ee695e78937de01b7128d3866b8fc5dd328ebdf2c159be37918d4f02952d5248", "pre_mlp_input": "e53fa425d89dfeb0e9395eca9bcd73dc0a55314abf45c5b10b43e2cbfa724345" }, "logits_sha256": "bdacba5675f0579cec629bbf01b36c8660ad576cfdf9fef1bea8259eaeac9ec2", "loss_nats": 1.2989667654037476, "mixer_summary_sha256": "eb3ac0f07de9b755d6e67a510ab68d09d9ad666bb7b5d020ba7d8c5055c399b2" }, "loss_nats": 1.2989667654037476, "loss_scale": 1.0, "mixers": [ { "branch": "attention", "entropy_mean": -0.0, "group": 1, "latest_source_mass": 1.0, "layer": 1, "max_source_mass": 1.0, "mixer_index": 0, "normalized_entropy": 1.0, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 1.0, "median": 1.0, "p05": 1.0, "p95": 1.0 } ], "sources": 1 }, { "branch": "mlp", "entropy_mean": 0.22886337339878082, "group": 1, "latest_source_mass": 0.06350618600845337, "layer": 1, "max_source_mass": 0.9364938139915466, "mixer_index": 1, "normalized_entropy": 0.3301800538435402, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.9364938139915466, "median": 0.9452826976776123, "p05": 0.8725647330284119, "p95": 0.9712384939193726 }, { "label": "current_group_partial", "mean": 0.06350618600845337, "median": 0.0547172985970974, "p05": 0.028761589899659157, "p95": 0.12743529677391052 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.603729784488678, "group": 1, "latest_source_mass": 0.4133155345916748, "layer": 2, "max_source_mass": 0.5866844654083252, "mixer_index": 2, "normalized_entropy": 0.8709979661187783, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.5866844654083252, "median": 0.5734395384788513, "p05": 0.29813364148139954, "p95": 0.8843642473220825 }, { "label": "current_group_partial", "mean": 0.4133155345916748, "median": 0.4265604615211487, "p05": 0.11563576757907867, "p95": 0.7018663287162781 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.3967982828617096, "group": 1, "latest_source_mass": 0.14110636711120605, "layer": 2, "max_source_mass": 0.858893632888794, "mixer_index": 3, "normalized_entropy": 0.5724589149178446, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.858893632888794, "median": 0.8705973029136658, "p05": 0.7599387168884277, "p95": 0.9270997643470764 }, { "label": "current_group_partial", "mean": 0.14110636711120605, "median": 0.12940272688865662, "p05": 0.07290027290582657, "p95": 0.2400612235069275 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6852496862411499, "group": 1, "latest_source_mass": 0.518975019454956, "layer": 3, "max_source_mass": 0.518975019454956, "mixer_index": 4, "normalized_entropy": 0.9886063241108252, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.48102498054504395, "median": 0.48102718591690063, "p05": 0.38786420226097107, "p95": 0.580143928527832 }, { "label": "current_group_partial", "mean": 0.518975019454956, "median": 0.5189728140830994, "p05": 0.41985607147216797, "p95": 0.6121357679367065 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.5890734195709229, "group": 1, "latest_source_mass": 0.3123423159122467, "layer": 3, "max_source_mass": 0.6876576542854309, "mixer_index": 5, "normalized_entropy": 0.8498533011344741, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.6876576542854309, "median": 0.6909762620925903, "p05": 0.48803940415382385, "p95": 0.8900076746940613 }, { "label": "current_group_partial", "mean": 0.3123423159122467, "median": 0.3090237081050873, "p05": 0.10999235510826111, "p95": 0.5119606256484985 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.6873323321342468, "group": 1, "latest_source_mass": 0.4831485152244568, "layer": 4, "max_source_mass": 0.5168514251708984, "mixer_index": 6, "normalized_entropy": 0.9916109470127238, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.5168514251708984, "median": 0.5219584703445435, "p05": 0.4251725673675537, "p95": 0.5947866439819336 }, { "label": "current_group_partial", "mean": 0.4831485152244568, "median": 0.47804147005081177, "p05": 0.405213326215744, "p95": 0.5748274326324463 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.487301766872406, "group": 1, "latest_source_mass": 0.2060384750366211, "layer": 4, "max_source_mass": 0.7939615249633789, "mixer_index": 7, "normalized_entropy": 0.7030278424832499, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.7939615249633789, "median": 0.7953404188156128, "p05": 0.6590759754180908, "p95": 0.9329437017440796 }, { "label": "current_group_partial", "mean": 0.2060384750366211, "median": 0.20465964078903198, "p05": 0.06705628335475922, "p95": 0.3409239649772644 } ], "sources": 2 }, { "branch": "attention", "entropy_mean": 0.674697756767273, "group": 2, "latest_source_mass": 0.42952626943588257, "layer": 5, "max_source_mass": 0.5704737305641174, "mixer_index": 8, "normalized_entropy": 0.9733831077870527, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.5704737305641174, "median": 0.5696132779121399, "p05": 0.4695202708244324, "p95": 0.6750621199607849 }, { "label": "completed_group_1", "mean": 0.42952626943588257, "median": 0.4303867816925049, "p05": 0.3249378800392151, "p95": 0.5304797887802124 } ], "sources": 2 }, { "branch": "mlp", "entropy_mean": 0.9691112041473389, "group": 2, "latest_source_mass": 0.2681158185005188, "layer": 5, "max_source_mass": 0.5506307482719421, "mixer_index": 9, "normalized_entropy": 0.8821230329784768, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.5506307482719421, "median": 0.543993353843689, "p05": 0.41262108087539673, "p95": 0.7161091566085815 }, { "label": "completed_group_1", "mean": 0.18125343322753906, "median": 0.1794438362121582, "p05": 0.11575750261545181, "p95": 0.25031501054763794 }, { "label": "current_group_partial", "mean": 0.2681158185005188, "median": 0.2664111256599426, "p05": 0.1564716249704361, "p95": 0.3817881643772125 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0745112895965576, "group": 2, "latest_source_mass": 0.412794291973114, "layer": 6, "max_source_mass": 0.412794291973114, "mixer_index": 10, "normalized_entropy": 0.9780623252441762, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.34262996912002563, "median": 0.3412924110889435, "p05": 0.29556700587272644, "p95": 0.39125555753707886 }, { "label": "completed_group_1", "mean": 0.24457570910453796, "median": 0.24382859468460083, "p05": 0.22120657563209534, "p95": 0.2734297513961792 }, { "label": "current_group_partial", "mean": 0.412794291973114, "median": 0.4151499271392822, "p05": 0.37735897302627563, "p95": 0.43749260902404785 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0074045658111572, "group": 2, "latest_source_mass": 0.3775783181190491, "layer": 6, "max_source_mass": 0.4399106502532959, "mixer_index": 11, "normalized_entropy": 0.9169791528842925, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.4399106502532959, "median": 0.44856905937194824, "p05": 0.22063353657722473, "p95": 0.6359403133392334 }, { "label": "completed_group_1", "mean": 0.18251106142997742, "median": 0.18562838435173035, "p05": 0.1196163073182106, "p95": 0.23045453429222107 }, { "label": "current_group_partial", "mean": 0.3775783181190491, "median": 0.3675365447998047, "p05": 0.23817461729049683, "p95": 0.5557451844215393 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.060497522354126, "group": 2, "latest_source_mass": 0.4025024175643921, "layer": 7, "max_source_mass": 0.4025024175643921, "mixer_index": 12, "normalized_entropy": 0.9653064445872968, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.3786153197288513, "median": 0.3907395303249359, "p05": 0.28039106726646423, "p95": 0.4436405599117279 }, { "label": "completed_group_1", "mean": 0.2188822478055954, "median": 0.21226292848587036, "p05": 0.1885872334241867, "p95": 0.2672030031681061 }, { "label": "current_group_partial", "mean": 0.4025024175643921, "median": 0.39815032482147217, "p05": 0.3533596992492676, "p95": 0.460100919008255 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 0.9941019415855408, "group": 2, "latest_source_mass": 0.4801652431488037, "layer": 7, "max_source_mass": 0.4801652431488037, "mixer_index": 13, "normalized_entropy": 0.9048705824970601, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.32296955585479736, "median": 0.2803739905357361, "p05": 0.1784922033548355, "p95": 0.6440245509147644 }, { "label": "completed_group_1", "mean": 0.19686520099639893, "median": 0.2053825855255127, "p05": 0.09813719242811203, "p95": 0.2504696547985077 }, { "label": "current_group_partial", "mean": 0.4801652431488037, "median": 0.5057140588760376, "p05": 0.2529003322124481, "p95": 0.598870575428009 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0503522157669067, "group": 2, "latest_source_mass": 0.4001518487930298, "layer": 8, "max_source_mass": 0.40130966901779175, "mixer_index": 14, "normalized_entropy": 0.9560717885654542, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.40130966901779175, "median": 0.396759569644928, "p05": 0.3479611873626709, "p95": 0.4733521044254303 }, { "label": "completed_group_1", "mean": 0.19853851199150085, "median": 0.19769755005836487, "p05": 0.17136667668819427, "p95": 0.22788912057876587 }, { "label": "current_group_partial", "mean": 0.4001518487930298, "median": 0.40291696786880493, "p05": 0.3506356179714203, "p95": 0.43941178917884827 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.0129402875900269, "group": 2, "latest_source_mass": 0.42161279916763306, "layer": 8, "max_source_mass": 0.42161279916763306, "mixer_index": 15, "normalized_entropy": 0.9220179839951123, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.4093267023563385, "median": 0.4046899676322937, "p05": 0.2895161807537079, "p95": 0.5762131810188293 }, { "label": "completed_group_1", "mean": 0.16906049847602844, "median": 0.16888955235481262, "p05": 0.11233551055192947, "p95": 0.22477403283119202 }, { "label": "current_group_partial", "mean": 0.42161279916763306, "median": 0.4227708578109741, "p05": 0.3131049573421478, "p95": 0.519075334072113 } ], "sources": 3 }, { "branch": "attention", "entropy_mean": 1.0774019956588745, "group": 3, "latest_source_mass": 0.3560434877872467, "layer": 9, "max_source_mass": 0.38510820269584656, "mixer_index": 16, "normalized_entropy": 0.9806935592947451, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.38510820269584656, "median": 0.3892287313938141, "p05": 0.2934730052947998, "p95": 0.4718902111053467 }, { "label": "completed_group_1", "mean": 0.25884830951690674, "median": 0.25734007358551025, "p05": 0.22217319905757904, "p95": 0.29551801085472107 }, { "label": "completed_group_2", "mean": 0.3560434877872467, "median": 0.35412120819091797, "p05": 0.29444968700408936, "p95": 0.41908568143844604 } ], "sources": 3 }, { "branch": "mlp", "entropy_mean": 1.2740377187728882, "group": 3, "latest_source_mass": 0.3633861243724823, "layer": 9, "max_source_mass": 0.3633861243724823, "mixer_index": 17, "normalized_entropy": 0.9190239493895668, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.29550763964653015, "median": 0.30360478162765503, "p05": 0.16904757916927338, "p95": 0.45146965980529785 }, { "label": "completed_group_1", "mean": 0.09541244804859161, "median": 0.09486871212720871, "p05": 0.061360105872154236, "p95": 0.13037073612213135 }, { "label": "completed_group_2", "mean": 0.24569380283355713, "median": 0.24333015084266663, "p05": 0.1698218435049057, "p95": 0.31589236855506897 }, { "label": "current_group_partial", "mean": 0.3633861243724823, "median": 0.36472851037979126, "p05": 0.2795379161834717, "p95": 0.4399876892566681 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.360302448272705, "group": 3, "latest_source_mass": 0.28569501638412476, "layer": 10, "max_source_mass": 0.3109527826309204, "mixer_index": 18, "normalized_entropy": 0.9812507981160736, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.3109527826309204, "median": 0.32172977924346924, "p05": 0.23520725965499878, "p95": 0.37052470445632935 }, { "label": "completed_group_1", "mean": 0.20984819531440735, "median": 0.20958667993545532, "p05": 0.1891367882490158, "p95": 0.23184512555599213 }, { "label": "completed_group_2", "mean": 0.19350400567054749, "median": 0.19153150916099548, "p05": 0.16607457399368286, "p95": 0.22877082228660583 }, { "label": "current_group_partial", "mean": 0.28569501638412476, "median": 0.28060320019721985, "p05": 0.23824527859687805, "p95": 0.3439165949821472 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.2722704410552979, "group": 3, "latest_source_mass": 0.29912811517715454, "layer": 10, "max_source_mass": 0.35036465525627136, "mixer_index": 19, "normalized_entropy": 0.9177491279900463, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.35036465525627136, "median": 0.36210548877716064, "p05": 0.12807878851890564, "p95": 0.5797606706619263 }, { "label": "completed_group_1", "mean": 0.12631990015506744, "median": 0.1264108121395111, "p05": 0.06977735459804535, "p95": 0.1755298376083374 }, { "label": "completed_group_2", "mean": 0.22418735921382904, "median": 0.22563719749450684, "p05": 0.1176326721906662, "p95": 0.32001155614852905 }, { "label": "current_group_partial", "mean": 0.29912811517715454, "median": 0.28589946031570435, "p05": 0.193109393119812, "p95": 0.44119665026664734 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3558036088943481, "group": 3, "latest_source_mass": 0.314197838306427, "layer": 11, "max_source_mass": 0.314197838306427, "mixer_index": 20, "normalized_entropy": 0.978005571485618, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.298976868391037, "median": 0.2961939573287964, "p05": 0.24950163066387177, "p95": 0.35835936665534973 }, { "label": "completed_group_1", "mean": 0.1941717267036438, "median": 0.1918032318353653, "p05": 0.16809208691120148, "p95": 0.22895419597625732 }, { "label": "completed_group_2", "mean": 0.1926535815000534, "median": 0.19306811690330505, "p05": 0.16983944177627563, "p95": 0.2155935913324356 }, { "label": "current_group_partial", "mean": 0.314197838306427, "median": 0.31541913747787476, "p05": 0.2673022747039795, "p95": 0.35941022634506226 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.2696614265441895, "group": 3, "latest_source_mass": 0.2862822115421295, "layer": 11, "max_source_mass": 0.3667730689048767, "mixer_index": 21, "normalized_entropy": 0.9158671218416545, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.3667730689048767, "median": 0.4067659378051758, "p05": 0.10593366622924805, "p95": 0.5672274827957153 }, { "label": "completed_group_1", "mean": 0.15310388803482056, "median": 0.15050218999385834, "p05": 0.08651046454906464, "p95": 0.22459912300109863 }, { "label": "completed_group_2", "mean": 0.19384080171585083, "median": 0.18732509016990662, "p05": 0.1267009675502777, "p95": 0.281107634305954 }, { "label": "current_group_partial", "mean": 0.2862822115421295, "median": 0.26271265745162964, "p05": 0.17024393379688263, "p95": 0.475044310092926 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3606936931610107, "group": 3, "latest_source_mass": 0.3028184771537781, "layer": 12, "max_source_mass": 0.3028184771537781, "mixer_index": 22, "normalized_entropy": 0.9815330216461395, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.298227459192276, "median": 0.2954302132129669, "p05": 0.25287362933158875, "p95": 0.35405808687210083 }, { "label": "completed_group_1", "mean": 0.18611131608486176, "median": 0.1850002110004425, "p05": 0.15901757776737213, "p95": 0.21725836396217346 }, { "label": "completed_group_2", "mean": 0.21284276247024536, "median": 0.21402311325073242, "p05": 0.18819917738437653, "p95": 0.23418807983398438 }, { "label": "current_group_partial", "mean": 0.3028184771537781, "median": 0.3041122555732727, "p05": 0.25661808252334595, "p95": 0.34458184242248535 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.269316554069519, "group": 3, "latest_source_mass": 0.2653314769268036, "layer": 12, "max_source_mass": 0.4081852436065674, "mixer_index": 23, "normalized_entropy": 0.9156183489371815, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.4081852436065674, "median": 0.4254342317581177, "p05": 0.20765496790409088, "p95": 0.587238073348999 }, { "label": "completed_group_1", "mean": 0.14420127868652344, "median": 0.1438048928976059, "p05": 0.1064920499920845, "p95": 0.1871720254421234 }, { "label": "completed_group_2", "mean": 0.18228203058242798, "median": 0.17507661879062653, "p05": 0.12183455377817154, "p95": 0.26505547761917114 }, { "label": "current_group_partial", "mean": 0.2653314769268036, "median": 0.24924281239509583, "p05": 0.15609969198703766, "p95": 0.4061311185359955 } ], "sources": 4 }, { "branch": "attention", "entropy_mean": 1.3636136054992676, "group": 4, "latest_source_mass": 0.3136237859725952, "layer": 13, "max_source_mass": 0.3136237859725952, "mixer_index": 24, "normalized_entropy": 0.9836392931712563, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.27427661418914795, "median": 0.27403637766838074, "p05": 0.22928935289382935, "p95": 0.319773405790329 }, { "label": "completed_group_1", "mean": 0.21341991424560547, "median": 0.20697325468063354, "p05": 0.18552586436271667, "p95": 0.26688331365585327 }, { "label": "completed_group_2", "mean": 0.19867971539497375, "median": 0.197545126080513, "p05": 0.1794913411140442, "p95": 0.222100168466568 }, { "label": "completed_group_3", "mean": 0.3136237859725952, "median": 0.31875866651535034, "p05": 0.24156072735786438, "p95": 0.3695686459541321 } ], "sources": 4 }, { "branch": "mlp", "entropy_mean": 1.570773959159851, "group": 4, "latest_source_mass": 0.2294243425130844, "layer": 13, "max_source_mass": 0.2658740282058716, "mixer_index": 25, "normalized_entropy": 0.9759767351225285, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.2658740282058716, "median": 0.2747163474559784, "p05": 0.17927880585193634, "p95": 0.3512483835220337 }, { "label": "completed_group_1", "mean": 0.13419361412525177, "median": 0.13374894857406616, "p05": 0.11072897166013718, "p95": 0.1586270034313202 }, { "label": "completed_group_2", "mean": 0.1907704919576645, "median": 0.19074805080890656, "p05": 0.14699944853782654, "p95": 0.23316305875778198 }, { "label": "completed_group_3", "mean": 0.17973749339580536, "median": 0.17616114020347595, "p05": 0.1363743096590042, "p95": 0.23202598094940186 }, { "label": "current_group_partial", "mean": 0.2294243425130844, "median": 0.22995705902576447, "p05": 0.17975632846355438, "p95": 0.2789157032966614 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.5560038089752197, "group": 4, "latest_source_mass": 0.32468360662460327, "layer": 14, "max_source_mass": 0.32468360662460327, "mixer_index": 26, "normalized_entropy": 0.9667995248241249, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.20477107167243958, "median": 0.206403911113739, "p05": 0.16619175672531128, "p95": 0.2431035190820694 }, { "label": "completed_group_1", "mean": 0.15721669793128967, "median": 0.15658551454544067, "p05": 0.13878680765628815, "p95": 0.17778633534908295 }, { "label": "completed_group_2", "mean": 0.13811248540878296, "median": 0.13743364810943604, "p05": 0.12352626770734787, "p95": 0.15553249418735504 }, { "label": "completed_group_3", "mean": 0.1752161681652069, "median": 0.17309701442718506, "p05": 0.15223677456378937, "p95": 0.20474471151828766 }, { "label": "current_group_partial", "mean": 0.32468360662460327, "median": 0.32497870922088623, "p05": 0.2946181893348694, "p95": 0.35203880071640015 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.5659973621368408, "group": 4, "latest_source_mass": 0.26703086495399475, "layer": 14, "max_source_mass": 0.26703086495399475, "mixer_index": 27, "normalized_entropy": 0.9730088685238187, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.20531430840492249, "median": 0.20229148864746094, "p05": 0.10698921978473663, "p95": 0.3393132984638214 }, { "label": "completed_group_1", "mean": 0.15986156463623047, "median": 0.15952220559120178, "p05": 0.11969029903411865, "p95": 0.20180393755435944 }, { "label": "completed_group_2", "mean": 0.21557100117206573, "median": 0.21654027700424194, "p05": 0.1673085242509842, "p95": 0.2610440254211426 }, { "label": "completed_group_3", "mean": 0.15222224593162537, "median": 0.1510905921459198, "p05": 0.11800260841846466, "p95": 0.18893754482269287 }, { "label": "current_group_partial", "mean": 0.26703086495399475, "median": 0.258539617061615, "p05": 0.19330690801143646, "p95": 0.36521556973457336 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.5552496910095215, "group": 4, "latest_source_mass": 0.31774693727493286, "layer": 15, "max_source_mass": 0.31774693727493286, "mixer_index": 28, "normalized_entropy": 0.9663309649872576, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.2162160575389862, "median": 0.2145705670118332, "p05": 0.17727568745613098, "p95": 0.26878347992897034 }, { "label": "completed_group_1", "mean": 0.17982497811317444, "median": 0.18024154007434845, "p05": 0.15453973412513733, "p95": 0.2030555009841919 }, { "label": "completed_group_2", "mean": 0.1307673156261444, "median": 0.13107559084892273, "p05": 0.11463287472724915, "p95": 0.14630235731601715 }, { "label": "completed_group_3", "mean": 0.1554446816444397, "median": 0.15516987442970276, "p05": 0.13627474009990692, "p95": 0.1754152774810791 }, { "label": "current_group_partial", "mean": 0.31774693727493286, "median": 0.319039523601532, "p05": 0.27606821060180664, "p95": 0.35650834441185 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.5068919658660889, "group": 4, "latest_source_mass": 0.3298327624797821, "layer": 15, "max_source_mass": 0.3298327624797821, "mixer_index": 29, "normalized_entropy": 0.9362846209998111, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.20646697282791138, "median": 0.18775638937950134, "p05": 0.0806753933429718, "p95": 0.4274584949016571 }, { "label": "completed_group_1", "mean": 0.1197461187839508, "median": 0.11989086866378784, "p05": 0.0809398666024208, "p95": 0.15743419528007507 }, { "label": "completed_group_2", "mean": 0.19215428829193115, "median": 0.19620640575885773, "p05": 0.1148109883069992, "p95": 0.2491212636232376 }, { "label": "completed_group_3", "mean": 0.15179988741874695, "median": 0.15212899446487427, "p05": 0.10697861760854721, "p95": 0.19542087614536285 }, { "label": "current_group_partial", "mean": 0.3298327624797821, "median": 0.32837340235710144, "p05": 0.22746729850769043, "p95": 0.4339301884174347 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.5435223579406738, "group": 4, "latest_source_mass": 0.3224271535873413, "layer": 16, "max_source_mass": 0.3224271535873413, "mixer_index": 30, "normalized_entropy": 0.9590443632623663, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.22410975396633148, "median": 0.22760415077209473, "p05": 0.14593973755836487, "p95": 0.30093055963516235 }, { "label": "completed_group_1", "mean": 0.16986176371574402, "median": 0.16963240504264832, "p05": 0.13569067418575287, "p95": 0.20651353895664215 }, { "label": "completed_group_2", "mean": 0.13080403208732605, "median": 0.13075244426727295, "p05": 0.10945598781108856, "p95": 0.15187400579452515 }, { "label": "completed_group_3", "mean": 0.15279728174209595, "median": 0.15036612749099731, "p05": 0.12810495495796204, "p95": 0.18468116223812103 }, { "label": "current_group_partial", "mean": 0.3224271535873413, "median": 0.3164973258972168, "p05": 0.2600978910923004, "p95": 0.40530598163604736 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.4949920177459717, "group": 4, "latest_source_mass": 0.31847020983695984, "layer": 16, "max_source_mass": 0.31847020983695984, "mixer_index": 31, "normalized_entropy": 0.9288907675133354, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.27174079418182373, "median": 0.2585451304912567, "p05": 0.13639721274375916, "p95": 0.4698348045349121 }, { "label": "completed_group_1", "mean": 0.12178683280944824, "median": 0.11714950203895569, "p05": 0.08594125509262085, "p95": 0.1723237782716751 }, { "label": "completed_group_2", "mean": 0.1529826819896698, "median": 0.15223106741905212, "p05": 0.10371473431587219, "p95": 0.203575000166893 }, { "label": "completed_group_3", "mean": 0.1350194811820984, "median": 0.13363957405090332, "p05": 0.08964549005031586, "p95": 0.18468207120895386 }, { "label": "current_group_partial", "mean": 0.31847020983695984, "median": 0.3185216188430786, "p05": 0.21444711089134216, "p95": 0.4301546812057495 } ], "sources": 5 }, { "branch": "attention", "entropy_mean": 1.544264793395996, "group": 5, "latest_source_mass": 0.2861037850379944, "layer": 17, "max_source_mass": 0.2861037850379944, "mixer_index": 32, "normalized_entropy": 0.9595056643474137, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.24016115069389343, "median": 0.23868846893310547, "p05": 0.18435582518577576, "p95": 0.2983453571796417 }, { "label": "completed_group_1", "mean": 0.20730282366275787, "median": 0.1998077780008316, "p05": 0.16255050897598267, "p95": 0.28107208013534546 }, { "label": "completed_group_2", "mean": 0.09692573547363281, "median": 0.0967404842376709, "p05": 0.08098706603050232, "p95": 0.11221462488174438 }, { "label": "completed_group_3", "mean": 0.1695065200328827, "median": 0.1696585714817047, "p05": 0.136396586894989, "p95": 0.20130634307861328 }, { "label": "completed_group_4", "mean": 0.2861037850379944, "median": 0.28883200883865356, "p05": 0.22401469945907593, "p95": 0.3409370183944702 } ], "sources": 5 }, { "branch": "mlp", "entropy_mean": 1.686682939529419, "group": 5, "latest_source_mass": 0.12281867861747742, "layer": 17, "max_source_mass": 0.2885604202747345, "mixer_index": 33, "normalized_entropy": 0.9413556721740635, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.16755425930023193, "median": 0.1327776312828064, "p05": 0.0790192037820816, "p95": 0.3667412996292114 }, { "label": "completed_group_1", "mean": 0.10962115228176117, "median": 0.10525335371494293, "p05": 0.07840172946453094, "p95": 0.15405339002609253 }, { "label": "completed_group_2", "mean": 0.1783454418182373, "median": 0.17896884679794312, "p05": 0.10802385956048965, "p95": 0.24048911035060883 }, { "label": "completed_group_3", "mean": 0.13310004770755768, "median": 0.13033035397529602, "p05": 0.0904831737279892, "p95": 0.17934636771678925 }, { "label": "completed_group_4", "mean": 0.2885604202747345, "median": 0.2831825315952301, "p05": 0.16967125236988068, "p95": 0.4189499616622925 }, { "label": "current_group_partial", "mean": 0.12281867861747742, "median": 0.12062528729438782, "p05": 0.07461707293987274, "p95": 0.1782219111919403 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.7070834636688232, "group": 5, "latest_source_mass": 0.2988377809524536, "layer": 18, "max_source_mass": 0.2988377809524536, "mixer_index": 34, "normalized_entropy": 0.9527414214834803, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.181177020072937, "median": 0.18419909477233887, "p05": 0.13624237477779388, "p95": 0.22192169725894928 }, { "label": "completed_group_1", "mean": 0.14557164907455444, "median": 0.144476979970932, "p05": 0.1293315887451172, "p95": 0.16486899554729462 }, { "label": "completed_group_2", "mean": 0.08318156749010086, "median": 0.0820261612534523, "p05": 0.07088497281074524, "p95": 0.0990293100476265 }, { "label": "completed_group_3", "mean": 0.11424209177494049, "median": 0.11267061531543732, "p05": 0.0979941189289093, "p95": 0.1353992223739624 }, { "label": "completed_group_4", "mean": 0.17698991298675537, "median": 0.17597860097885132, "p05": 0.14837387204170227, "p95": 0.21050144731998444 }, { "label": "current_group_partial", "mean": 0.2988377809524536, "median": 0.2989804744720459, "p05": 0.2721370458602905, "p95": 0.32572072744369507 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.7419235706329346, "group": 5, "latest_source_mass": 0.26016175746917725, "layer": 18, "max_source_mass": 0.26016175746917725, "mixer_index": 35, "normalized_entropy": 0.972186055410333, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.1532188355922699, "median": 0.1451343297958374, "p05": 0.11514341086149216, "p95": 0.22103679180145264 }, { "label": "completed_group_1", "mean": 0.11420352011919022, "median": 0.11261284351348877, "p05": 0.0928458571434021, "p95": 0.14123988151550293 }, { "label": "completed_group_2", "mean": 0.16597424447536469, "median": 0.1655328869819641, "p05": 0.13871800899505615, "p95": 0.19611573219299316 }, { "label": "completed_group_3", "mean": 0.1250365972518921, "median": 0.12369821965694427, "p05": 0.10375828295946121, "p95": 0.14912742376327515 }, { "label": "completed_group_4", "mean": 0.18140508234500885, "median": 0.1796584278345108, "p05": 0.14511024951934814, "p95": 0.22286373376846313 }, { "label": "current_group_partial", "mean": 0.26016175746917725, "median": 0.26266711950302124, "p05": 0.1889389455318451, "p95": 0.3234381675720215 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.7127439975738525, "group": 5, "latest_source_mass": 0.2907913327217102, "layer": 19, "max_source_mass": 0.2907913327217102, "mixer_index": 36, "normalized_entropy": 0.9559006256078307, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.18731768429279327, "median": 0.19042012095451355, "p05": 0.13879702985286713, "p95": 0.23792000114917755 }, { "label": "completed_group_1", "mean": 0.16715484857559204, "median": 0.1667211651802063, "p05": 0.14574480056762695, "p95": 0.19026139378547668 }, { "label": "completed_group_2", "mean": 0.08496320247650146, "median": 0.08452782034873962, "p05": 0.07410907745361328, "p95": 0.097386933863163 }, { "label": "completed_group_3", "mean": 0.11362075805664062, "median": 0.11248823255300522, "p05": 0.0977015346288681, "p95": 0.13219426572322845 }, { "label": "completed_group_4", "mean": 0.15615218877792358, "median": 0.15540224313735962, "p05": 0.1289147436618805, "p95": 0.18598109483718872 }, { "label": "current_group_partial", "mean": 0.2907913327217102, "median": 0.2900415062904358, "p05": 0.25822532176971436, "p95": 0.32475030422210693 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.6076834201812744, "group": 5, "latest_source_mass": 0.3738824725151062, "layer": 19, "max_source_mass": 0.3738824725151062, "mixer_index": 37, "normalized_entropy": 0.8972652009334232, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.15032559633255005, "median": 0.12544110417366028, "p05": 0.06976518034934998, "p95": 0.35840943455696106 }, { "label": "completed_group_1", "mean": 0.08624927699565887, "median": 0.08602266013622284, "p05": 0.061998721212148666, "p95": 0.11138717085123062 }, { "label": "completed_group_2", "mean": 0.15736976265907288, "median": 0.15802210569381714, "p05": 0.10121746361255646, "p95": 0.20687000453472137 }, { "label": "completed_group_3", "mean": 0.08584291487932205, "median": 0.0827617198228836, "p05": 0.060561805963516235, "p95": 0.11351694166660309 }, { "label": "completed_group_4", "mean": 0.14632999897003174, "median": 0.1426895558834076, "p05": 0.10106660425662994, "p95": 0.20179018378257751 }, { "label": "current_group_partial", "mean": 0.3738824725151062, "median": 0.3799295425415039, "p05": 0.23631010949611664, "p95": 0.4884527325630188 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.709290862083435, "group": 5, "latest_source_mass": 0.2901708781719208, "layer": 20, "max_source_mass": 0.2901708781719208, "mixer_index": 38, "normalized_entropy": 0.9539733939957076, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.18991519510746002, "median": 0.1924753338098526, "p05": 0.14201581478118896, "p95": 0.23709386587142944 }, { "label": "completed_group_1", "mean": 0.17091524600982666, "median": 0.17006167769432068, "p05": 0.15297070145606995, "p95": 0.19153237342834473 }, { "label": "completed_group_2", "mean": 0.07822999358177185, "median": 0.07778562605381012, "p05": 0.06707640737295151, "p95": 0.09089364111423492 }, { "label": "completed_group_3", "mean": 0.116765096783638, "median": 0.11516137421131134, "p05": 0.10162825882434845, "p95": 0.13653048872947693 }, { "label": "completed_group_4", "mean": 0.1540035903453827, "median": 0.15287795662879944, "p05": 0.12593276798725128, "p95": 0.18590238690376282 }, { "label": "current_group_partial", "mean": 0.2901708781719208, "median": 0.29104921221733093, "p05": 0.25643545389175415, "p95": 0.3205650746822357 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.6690887212753296, "group": 5, "latest_source_mass": 0.3342471718788147, "layer": 20, "max_source_mass": 0.3342471718788147, "mixer_index": 39, "normalized_entropy": 0.9315361520005944, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.19029226899147034, "median": 0.19644704461097717, "p05": 0.09456793963909149, "p95": 0.30468082427978516 }, { "label": "completed_group_1", "mean": 0.1100241169333458, "median": 0.10808776319026947, "p05": 0.09035106748342514, "p95": 0.1359245330095291 }, { "label": "completed_group_2", "mean": 0.11471512168645859, "median": 0.11494918912649155, "p05": 0.08799737691879272, "p95": 0.14024731516838074 }, { "label": "completed_group_3", "mean": 0.09999909996986389, "median": 0.09682698547840118, "p05": 0.07921088486909866, "p95": 0.1264391839504242 }, { "label": "completed_group_4", "mean": 0.1507222205400467, "median": 0.1476232260465622, "p05": 0.11397356539964676, "p95": 0.19991227984428406 }, { "label": "current_group_partial", "mean": 0.3342471718788147, "median": 0.33802270889282227, "p05": 0.2548266053199768, "p95": 0.40130624175071716 } ], "sources": 6 }, { "branch": "attention", "entropy_mean": 1.7458693981170654, "group": 6, "latest_source_mass": 0.2361394464969635, "layer": 21, "max_source_mass": 0.2361394464969635, "mixer_index": 40, "normalized_entropy": 0.9743882636597644, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.20528709888458252, "median": 0.2140858769416809, "p05": 0.13168928027153015, "p95": 0.27098023891448975 }, { "label": "completed_group_1", "mean": 0.1722964197397232, "median": 0.17238952219486237, "p05": 0.15053534507751465, "p95": 0.19365043938159943 }, { "label": "completed_group_2", "mean": 0.10435444861650467, "median": 0.10361101478338242, "p05": 0.08888213336467743, "p95": 0.12212536484003067 }, { "label": "completed_group_3", "mean": 0.12075909972190857, "median": 0.1180175170302391, "p05": 0.10358668118715286, "p95": 0.14458929002285004 }, { "label": "completed_group_4", "mean": 0.16116347908973694, "median": 0.15841388702392578, "p05": 0.1337081342935562, "p95": 0.19791515171527863 }, { "label": "completed_group_5", "mean": 0.2361394464969635, "median": 0.23709020018577576, "p05": 0.20446214079856873, "p95": 0.26509594917297363 } ], "sources": 6 }, { "branch": "mlp", "entropy_mean": 1.4612125158309937, "group": 6, "latest_source_mass": 0.5543455481529236, "layer": 21, "max_source_mass": 0.5543455481529236, "mixer_index": 41, "normalized_entropy": 0.7509146897354808, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.08934466540813446, "median": 0.07911523431539536, "p05": 0.04488275945186615, "p95": 0.1737232506275177 }, { "label": "completed_group_1", "mean": 0.054379627108573914, "median": 0.05344422906637192, "p05": 0.04221750423312187, "p95": 0.06924191862344742 }, { "label": "completed_group_2", "mean": 0.07567165046930313, "median": 0.07438549399375916, "p05": 0.05811355262994766, "p95": 0.09632256627082825 }, { "label": "completed_group_3", "mean": 0.05758609622716904, "median": 0.05612001568078995, "p05": 0.044792283326387405, "p95": 0.07347827404737473 }, { "label": "completed_group_4", "mean": 0.07884950935840607, "median": 0.07757881283760071, "p05": 0.05758974328637123, "p95": 0.10452556610107422 }, { "label": "completed_group_5", "mean": 0.08982288837432861, "median": 0.08912864327430725, "p05": 0.0663854256272316, "p95": 0.11604885756969452 }, { "label": "current_group_partial", "mean": 0.5543455481529236, "median": 0.5596089363098145, "p05": 0.45455610752105713, "p95": 0.6367484331130981 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.7241814136505127, "group": 6, "latest_source_mass": 0.39484071731567383, "layer": 22, "max_source_mass": 0.39484071731567383, "mixer_index": 42, "normalized_entropy": 0.886053970419732, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.13360010087490082, "median": 0.14072942733764648, "p05": 0.07699814438819885, "p95": 0.17506597936153412 }, { "label": "completed_group_1", "mean": 0.10609637200832367, "median": 0.10586297512054443, "p05": 0.08648421615362167, "p95": 0.12602579593658447 }, { "label": "completed_group_2", "mean": 0.06363911926746368, "median": 0.06324908882379532, "p05": 0.054980404675006866, "p95": 0.07316673547029495 }, { "label": "completed_group_3", "mean": 0.07426095008850098, "median": 0.07362888753414154, "p05": 0.06318790465593338, "p95": 0.08665015548467636 }, { "label": "completed_group_4", "mean": 0.09383076429367065, "median": 0.09294568747282028, "p05": 0.07638302445411682, "p95": 0.11445100605487823 }, { "label": "completed_group_5", "mean": 0.13373197615146637, "median": 0.13327741622924805, "p05": 0.11399447172880173, "p95": 0.15641780197620392 }, { "label": "current_group_partial", "mean": 0.39484071731567383, "median": 0.3879151940345764, "p05": 0.3302172124385834, "p95": 0.4852263331413269 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.3979300260543823, "group": 6, "latest_source_mass": 0.5830056071281433, "layer": 22, "max_source_mass": 0.5830056071281433, "mixer_index": 43, "normalized_entropy": 0.7183939231382495, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.08959861099720001, "median": 0.07544538378715515, "p05": 0.04165133833885193, "p95": 0.20099139213562012 }, { "label": "completed_group_1", "mean": 0.05414731800556183, "median": 0.05336771905422211, "p05": 0.04069708660244942, "p95": 0.07046990841627121 }, { "label": "completed_group_2", "mean": 0.07668881118297577, "median": 0.07607124000787735, "p05": 0.0555989071726799, "p95": 0.09980960935354233 }, { "label": "completed_group_3", "mean": 0.05603286623954773, "median": 0.054573021829128265, "p05": 0.04162490367889404, "p95": 0.0730118378996849 }, { "label": "completed_group_4", "mean": 0.06979228556156158, "median": 0.06884932518005371, "p05": 0.04576495662331581, "p95": 0.09806846082210541 }, { "label": "completed_group_5", "mean": 0.07073447108268738, "median": 0.07023533433675766, "p05": 0.04956750199198723, "p95": 0.09379896521568298 }, { "label": "current_group_partial", "mean": 0.5830056071281433, "median": 0.5862616896629333, "p05": 0.4658474922180176, "p95": 0.6849814057350159 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.7459560632705688, "group": 6, "latest_source_mass": 0.37691548466682434, "layer": 23, "max_source_mass": 0.37691548466682434, "mixer_index": 44, "normalized_entropy": 0.897243926765161, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.14318783581256866, "median": 0.1493408977985382, "p05": 0.08292607218027115, "p95": 0.19464610517024994 }, { "label": "completed_group_1", "mean": 0.1094653457403183, "median": 0.10858799517154694, "p05": 0.09432064741849899, "p95": 0.1279931515455246 }, { "label": "completed_group_2", "mean": 0.06349228322505951, "median": 0.06325754523277283, "p05": 0.05439673364162445, "p95": 0.0737837627530098 }, { "label": "completed_group_3", "mean": 0.07859592139720917, "median": 0.0775163471698761, "p05": 0.06603655964136124, "p95": 0.09357087314128876 }, { "label": "completed_group_4", "mean": 0.09567482024431229, "median": 0.09468592703342438, "p05": 0.07671141624450684, "p95": 0.11774884164333344 }, { "label": "completed_group_5", "mean": 0.13266831636428833, "median": 0.13199329376220703, "p05": 0.11244359612464905, "p95": 0.15504513680934906 }, { "label": "current_group_partial", "mean": 0.37691548466682434, "median": 0.37180638313293457, "p05": 0.31207239627838135, "p95": 0.4595233201980591 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.4055393934249878, "group": 6, "latest_source_mass": 0.5772827863693237, "layer": 23, "max_source_mass": 0.5772827863693237, "mixer_index": 45, "normalized_entropy": 0.7223043644164862, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.10154955089092255, "median": 0.08766824007034302, "p05": 0.04901552200317383, "p95": 0.20287738740444183 }, { "label": "completed_group_1", "mean": 0.05619276314973831, "median": 0.055477917194366455, "p05": 0.041608311235904694, "p95": 0.07371112704277039 }, { "label": "completed_group_2", "mean": 0.07791627198457718, "median": 0.07736755907535553, "p05": 0.055920060724020004, "p95": 0.10158214718103409 }, { "label": "completed_group_3", "mean": 0.053915880620479584, "median": 0.05215863138437271, "p05": 0.03798118978738785, "p95": 0.07156119495630264 }, { "label": "completed_group_4", "mean": 0.06398355215787888, "median": 0.06229078769683838, "p05": 0.040616974234580994, "p95": 0.09270931780338287 }, { "label": "completed_group_5", "mean": 0.06915920227766037, "median": 0.0684593990445137, "p05": 0.046986356377601624, "p95": 0.09408729523420334 }, { "label": "current_group_partial", "mean": 0.5772827863693237, "median": 0.5824689269065857, "p05": 0.4518115520477295, "p95": 0.6926202178001404 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.7853620052337646, "group": 6, "latest_source_mass": 0.3407441973686218, "layer": 24, "max_source_mass": 0.3407441973686218, "mixer_index": 46, "normalized_entropy": 0.9174945750195659, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.15284113585948944, "median": 0.1593906283378601, "p05": 0.08678612112998962, "p95": 0.211342453956604 }, { "label": "completed_group_1", "mean": 0.12206703424453735, "median": 0.12153112143278122, "p05": 0.1060115247964859, "p95": 0.1399766057729721 }, { "label": "completed_group_2", "mean": 0.06508444249629974, "median": 0.06482555717229843, "p05": 0.055350616574287415, "p95": 0.07557765394449234 }, { "label": "completed_group_3", "mean": 0.08347313106060028, "median": 0.08190013468265533, "p05": 0.06966532766819, "p95": 0.10050469636917114 }, { "label": "completed_group_4", "mean": 0.09811970591545105, "median": 0.09664031863212585, "p05": 0.07688035070896149, "p95": 0.12369741499423981 }, { "label": "completed_group_5", "mean": 0.13767032325267792, "median": 0.1373325139284134, "p05": 0.11642646044492722, "p95": 0.1606694608926773 }, { "label": "current_group_partial", "mean": 0.3407441973686218, "median": 0.33805108070373535, "p05": 0.271301805973053, "p95": 0.41781002283096313 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.509842872619629, "group": 6, "latest_source_mass": 0.5227814316749573, "layer": 24, "max_source_mass": 0.5227814316749573, "mixer_index": 47, "normalized_entropy": 0.7759057494780099, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.11788781732320786, "median": 0.10097194463014603, "p05": 0.06265684217214584, "p95": 0.2408325970172882 }, { "label": "completed_group_1", "mean": 0.06822571158409119, "median": 0.0673636943101883, "p05": 0.05209728330373764, "p95": 0.08648915588855743 }, { "label": "completed_group_2", "mean": 0.08556114137172699, "median": 0.08463940769433975, "p05": 0.06234566867351532, "p95": 0.1125786080956459 }, { "label": "completed_group_3", "mean": 0.06067090481519699, "median": 0.058974385261535645, "p05": 0.043267179280519485, "p95": 0.07997901737689972 }, { "label": "completed_group_4", "mean": 0.07228855788707733, "median": 0.06972275674343109, "p05": 0.04689513146877289, "p95": 0.106512151658535 }, { "label": "completed_group_5", "mean": 0.07258446514606476, "median": 0.07082539796829224, "p05": 0.050949547439813614, "p95": 0.09970614314079285 }, { "label": "current_group_partial", "mean": 0.5227814316749573, "median": 0.5290273427963257, "p05": 0.38773614168167114, "p95": 0.6392589807510376 } ], "sources": 7 }, { "branch": "attention", "entropy_mean": 1.839367389678955, "group": 7, "latest_source_mass": 0.29583215713500977, "layer": 25, "max_source_mass": 0.29583215713500977, "mixer_index": 48, "normalized_entropy": 0.9452478525649903, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.15906387567520142, "median": 0.15846745669841766, "p05": 0.09519976377487183, "p95": 0.24517810344696045 }, { "label": "completed_group_1", "mean": 0.13139380514621735, "median": 0.1308426856994629, "p05": 0.11478274315595627, "p95": 0.15000829100608826 }, { "label": "completed_group_2", "mean": 0.07717745751142502, "median": 0.07715500891208649, "p05": 0.06616044044494629, "p95": 0.08791667222976685 }, { "label": "completed_group_3", "mean": 0.09607303142547607, "median": 0.09436552226543427, "p05": 0.08152313530445099, "p95": 0.11384639143943787 }, { "label": "completed_group_4", "mean": 0.1107850894331932, "median": 0.1089654266834259, "p05": 0.08814431726932526, "p95": 0.139452263712883 }, { "label": "completed_group_5", "mean": 0.12967455387115479, "median": 0.1299237608909607, "p05": 0.10832332074642181, "p95": 0.15138982236385345 }, { "label": "completed_group_6", "mean": 0.29583215713500977, "median": 0.29647016525268555, "p05": 0.2276897132396698, "p95": 0.3604946434497833 } ], "sources": 7 }, { "branch": "mlp", "entropy_mean": 1.5841834545135498, "group": 7, "latest_source_mass": 0.40767741203308105, "layer": 25, "max_source_mass": 0.40767741203308105, "mixer_index": 49, "normalized_entropy": 0.7618312045616817, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.07896879315376282, "median": 0.06809917092323303, "p05": 0.025932887569069862, "p95": 0.19368338584899902 }, { "label": "completed_group_1", "mean": 0.048752427101135254, "median": 0.04767637699842453, "p05": 0.034443892538547516, "p95": 0.0667153000831604 }, { "label": "completed_group_2", "mean": 0.055707164108753204, "median": 0.0549345538020134, "p05": 0.038696639239788055, "p95": 0.0750277042388916 }, { "label": "completed_group_3", "mean": 0.04139826446771622, "median": 0.03878612071275711, "p05": 0.026673054322600365, "p95": 0.05746251344680786 }, { "label": "completed_group_4", "mean": 0.04449018836021423, "median": 0.04196460545063019, "p05": 0.026162121444940567, "p95": 0.07164917886257172 }, { "label": "completed_group_5", "mean": 0.04285799711942673, "median": 0.04109475016593933, "p05": 0.029393533244729042, "p95": 0.06277710199356079 }, { "label": "completed_group_6", "mean": 0.2801477909088135, "median": 0.2763082981109619, "p05": 0.1455708146095276, "p95": 0.4254714846611023 }, { "label": "current_group_partial", "mean": 0.40767741203308105, "median": 0.4120897948741913, "p05": 0.21399420499801636, "p95": 0.5853031277656555 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.8125314712524414, "group": 7, "latest_source_mass": 0.3700707256793976, "layer": 26, "max_source_mass": 0.3700707256793976, "mixer_index": 50, "normalized_entropy": 0.8716433883436914, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.10889513790607452, "median": 0.11333495378494263, "p05": 0.04850132763385773, "p95": 0.1620434820652008 }, { "label": "completed_group_1", "mean": 0.08656036853790283, "median": 0.08640051633119583, "p05": 0.0650845319032669, "p95": 0.1084962785243988 }, { "label": "completed_group_2", "mean": 0.048123255372047424, "median": 0.04800213500857353, "p05": 0.03576180338859558, "p95": 0.06107594817876816 }, { "label": "completed_group_3", "mean": 0.06561170518398285, "median": 0.06379580497741699, "p05": 0.050968848168849945, "p95": 0.08332888036966324 }, { "label": "completed_group_4", "mean": 0.06917259097099304, "median": 0.06724178791046143, "p05": 0.050207968801259995, "p95": 0.09373342245817184 }, { "label": "completed_group_5", "mean": 0.08286689221858978, "median": 0.08183546364307404, "p05": 0.06535022705793381, "p95": 0.10340933501720428 }, { "label": "completed_group_6", "mean": 0.16869932413101196, "median": 0.16509854793548584, "p05": 0.11059704422950745, "p95": 0.23654375970363617 }, { "label": "current_group_partial", "mean": 0.3700707256793976, "median": 0.3694133758544922, "p05": 0.23441238701343536, "p95": 0.5025027990341187 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.50089693069458, "group": 7, "latest_source_mass": 0.49096518754959106, "layer": 26, "max_source_mass": 0.49096518754959106, "mixer_index": 51, "normalized_entropy": 0.7217788529328457, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.07974998652935028, "median": 0.07402504235506058, "p05": 0.04579866677522659, "p95": 0.1359037309885025 }, { "label": "completed_group_1", "mean": 0.04166398197412491, "median": 0.04038450866937637, "p05": 0.027992289513349533, "p95": 0.05969971418380737 }, { "label": "completed_group_2", "mean": 0.04607272148132324, "median": 0.044438350945711136, "p05": 0.030038591474294662, "p95": 0.06808309257030487 }, { "label": "completed_group_3", "mean": 0.03586764633655548, "median": 0.03317485377192497, "p05": 0.02203454077243805, "p95": 0.05572832003235817 }, { "label": "completed_group_4", "mean": 0.04223340004682541, "median": 0.03930673003196716, "p05": 0.02465122565627098, "p95": 0.06968094408512115 }, { "label": "completed_group_5", "mean": 0.04095568507909775, "median": 0.03919431194663048, "p05": 0.026239857077598572, "p95": 0.06058088690042496 }, { "label": "completed_group_6", "mean": 0.22249138355255127, "median": 0.2125401496887207, "p05": 0.10647299885749817, "p95": 0.3711881935596466 }, { "label": "current_group_partial", "mean": 0.49096518754959106, "median": 0.4988659620285034, "p05": 0.2878011167049408, "p95": 0.6676807403564453 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.8059929609298706, "group": 7, "latest_source_mass": 0.37139976024627686, "layer": 27, "max_source_mass": 0.37139976024627686, "mixer_index": 52, "normalized_entropy": 0.8684990295379666, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.10711923241615295, "median": 0.10972198843955994, "p05": 0.04896187782287598, "p95": 0.16201043128967285 }, { "label": "completed_group_1", "mean": 0.07651707530021667, "median": 0.07666032761335373, "p05": 0.05141128972172737, "p95": 0.10152198374271393 }, { "label": "completed_group_2", "mean": 0.058284737169742584, "median": 0.05812416970729828, "p05": 0.039273619651794434, "p95": 0.07745087146759033 }, { "label": "completed_group_3", "mean": 0.06609587371349335, "median": 0.06383530795574188, "p05": 0.04481930285692215, "p95": 0.09336265176534653 }, { "label": "completed_group_4", "mean": 0.07139429450035095, "median": 0.0694514587521553, "p05": 0.0485839769244194, "p95": 0.10034126043319702 }, { "label": "completed_group_5", "mean": 0.07493151724338531, "median": 0.07319007813930511, "p05": 0.051876723766326904, "p95": 0.10402432829141617 }, { "label": "completed_group_6", "mean": 0.17425751686096191, "median": 0.16937679052352905, "p05": 0.10449051856994629, "p95": 0.2624811828136444 }, { "label": "current_group_partial", "mean": 0.37139976024627686, "median": 0.3732905089855194, "p05": 0.23830299079418182, "p95": 0.5014542937278748 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.5622732639312744, "group": 7, "latest_source_mass": 0.4768889844417572, "layer": 27, "max_source_mass": 0.4768889844417572, "mixer_index": 53, "normalized_entropy": 0.7512946301290215, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.08775827288627625, "median": 0.08504067361354828, "p05": 0.0428270623087883, "p95": 0.144285649061203 }, { "label": "completed_group_1", "mean": 0.047765426337718964, "median": 0.04646281152963638, "p05": 0.031814731657505035, "p95": 0.06807165592908859 }, { "label": "completed_group_2", "mean": 0.053928617388010025, "median": 0.052262187004089355, "p05": 0.035437777638435364, "p95": 0.0784272700548172 }, { "label": "completed_group_3", "mean": 0.04293380305171013, "median": 0.03996016085147858, "p05": 0.02556057646870613, "p95": 0.06623870134353638 }, { "label": "completed_group_4", "mean": 0.04174989461898804, "median": 0.03903091698884964, "p05": 0.025013400241732597, "p95": 0.06869364529848099 }, { "label": "completed_group_5", "mean": 0.04734005779027939, "median": 0.04609068110585213, "p05": 0.029884062707424164, "p95": 0.06911227852106094 }, { "label": "completed_group_6", "mean": 0.20163494348526, "median": 0.19222964346408844, "p05": 0.09963390231132507, "p95": 0.3307305574417114 }, { "label": "current_group_partial", "mean": 0.4768889844417572, "median": 0.48383551836013794, "p05": 0.276212215423584, "p95": 0.6487575173377991 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.8757455348968506, "group": 7, "latest_source_mass": 0.3109493851661682, "layer": 28, "max_source_mass": 0.3109493851661682, "mixer_index": 54, "normalized_entropy": 0.9020429270551009, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.11563669890165329, "median": 0.12087491154670715, "p05": 0.05733206868171692, "p95": 0.1688953936100006 }, { "label": "completed_group_1", "mean": 0.08108697831630707, "median": 0.08185897022485733, "p05": 0.05175650864839554, "p95": 0.10753051191568375 }, { "label": "completed_group_2", "mean": 0.07280858606100082, "median": 0.07357072830200195, "p05": 0.04870978742837906, "p95": 0.0948689877986908 }, { "label": "completed_group_3", "mean": 0.07553227245807648, "median": 0.07362619787454605, "p05": 0.056317005306482315, "p95": 0.09756095707416534 }, { "label": "completed_group_4", "mean": 0.07606014609336853, "median": 0.07456053793430328, "p05": 0.05528895929455757, "p95": 0.1022825762629509 }, { "label": "completed_group_5", "mean": 0.06935927271842957, "median": 0.06843917071819305, "p05": 0.04978285729885101, "p95": 0.09167985618114471 }, { "label": "completed_group_6", "mean": 0.19856664538383484, "median": 0.19508783519268036, "p05": 0.1281973123550415, "p95": 0.27796900272369385 }, { "label": "current_group_partial", "mean": 0.3109493851661682, "median": 0.3105033040046692, "p05": 0.18172834813594818, "p95": 0.43916669487953186 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.6924852132797241, "group": 7, "latest_source_mass": 0.3315581679344177, "layer": 28, "max_source_mass": 0.3315581679344177, "mixer_index": 55, "normalized_entropy": 0.8139133413255193, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.08638577908277512, "median": 0.058608733117580414, "p05": 0.026544082909822464, "p95": 0.2740764319896698 }, { "label": "completed_group_1", "mean": 0.059912845492362976, "median": 0.05901291221380234, "p05": 0.03629481792449951, "p95": 0.08707985281944275 }, { "label": "completed_group_2", "mean": 0.06609142571687698, "median": 0.0647939145565033, "p05": 0.038691695779561996, "p95": 0.09703366458415985 }, { "label": "completed_group_3", "mean": 0.056745611131191254, "median": 0.05418796092271805, "p05": 0.032174739986658096, "p95": 0.08370660990476608 }, { "label": "completed_group_4", "mean": 0.06277773529291153, "median": 0.0600711889564991, "p05": 0.03501531109213829, "p95": 0.10094858705997467 }, { "label": "completed_group_5", "mean": 0.05352145433425903, "median": 0.05179818347096443, "p05": 0.032271772623062134, "p95": 0.08084158599376678 }, { "label": "completed_group_6", "mean": 0.2830069661140442, "median": 0.2759752869606018, "p05": 0.12981083989143372, "p95": 0.46204638481140137 }, { "label": "current_group_partial", "mean": 0.3315581679344177, "median": 0.3204721510410309, "p05": 0.13446375727653503, "p95": 0.5667697787284851 } ], "sources": 8 }, { "branch": "attention", "entropy_mean": 1.8980522155761719, "group": 8, "latest_source_mass": 0.20123262703418732, "layer": 29, "max_source_mass": 0.26602888107299805, "mixer_index": 56, "normalized_entropy": 0.9127701729200177, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.11603894829750061, "median": 0.09991352260112762, "p05": 0.030035395175218582, "p95": 0.2941743731498718 }, { "label": "completed_group_1", "mean": 0.08804173022508621, "median": 0.08880250155925751, "p05": 0.05664174258708954, "p95": 0.11779069900512695 }, { "label": "completed_group_2", "mean": 0.07975024729967117, "median": 0.07984748482704163, "p05": 0.05428915470838547, "p95": 0.10499341040849686 }, { "label": "completed_group_3", "mean": 0.09027000516653061, "median": 0.08898856490850449, "p05": 0.06189651042222977, "p95": 0.11996553838253021 }, { "label": "completed_group_4", "mean": 0.08430293202400208, "median": 0.0819794088602066, "p05": 0.05464346706867218, "p95": 0.12159879505634308 }, { "label": "completed_group_5", "mean": 0.07433462142944336, "median": 0.07287181913852692, "p05": 0.052520088851451874, "p95": 0.1010315865278244 }, { "label": "completed_group_6", "mean": 0.26602888107299805, "median": 0.2639757990837097, "p05": 0.15474572777748108, "p95": 0.3846370279788971 }, { "label": "completed_group_7", "mean": 0.20123262703418732, "median": 0.18140269815921783, "p05": 0.08419936150312424, "p95": 0.3782624900341034 } ], "sources": 8 }, { "branch": "mlp", "entropy_mean": 1.7512003183364868, "group": 8, "latest_source_mass": 0.33482813835144043, "layer": 29, "max_source_mass": 0.33482813835144043, "mixer_index": 57, "normalized_entropy": 0.7970056117156376, "offset": 1, "source_summaries": [ { "label": "embedding", "mean": 0.07744461297988892, "median": 0.06970229744911194, "p05": 0.014401955530047417, "p95": 0.17144526541233063 }, { "label": "completed_group_1", "mean": 0.046086348593235016, "median": 0.04394831880927086, "p05": 0.028066720813512802, "p95": 0.0718703418970108 }, { "label": "completed_group_2", "mean": 0.05025433003902435, "median": 0.04861236736178398, "p05": 0.029644077643752098, "p95": 0.07761462777853012 }, { "label": "completed_group_3", "mean": 0.03748488053679466, "median": 0.03434944897890091, "p05": 0.020774798467755318, "p95": 0.05938909202814102 }, { "label": "completed_group_4", "mean": 0.04005061462521553, "median": 0.037498839199543, "p05": 0.019541850313544273, "p95": 0.06922487914562225 }, { "label": "completed_group_5", "mean": 0.041623204946517944, "median": 0.039552345871925354, "p05": 0.022081367671489716, "p95": 0.06821337342262268 }, { "label": "completed_group_6", "mean": 0.198423832654953, "median": 0.18318486213684082, "p05": 0.07548617571592331, "p95": 0.3721975088119507 }, { "label": "completed_group_7", "mean": 0.17380405962467194, "median": 0.1514919549226761, "p05": 0.05777408182621002, "p95": 0.36326974630355835 }, { "label": "current_group_partial", "mean": 0.33482813835144043, "median": 0.33602219820022583, "p05": 0.0795319601893425, "p95": 0.5979493856430054 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.0160694122314453, "group": 8, "latest_source_mass": 0.21711516380310059, "layer": 30, "max_source_mass": 0.21711516380310059, "mixer_index": 58, "normalized_entropy": 0.9175527313077866, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.09385271370410919, "median": 0.09211117774248123, "p05": 0.032377734780311584, "p95": 0.16496515274047852 }, { "label": "completed_group_1", "mean": 0.06208505108952522, "median": 0.06140740588307381, "p05": 0.0407925546169281, "p95": 0.0863042101264 }, { "label": "completed_group_2", "mean": 0.06560768187046051, "median": 0.06516212224960327, "p05": 0.041897669434547424, "p95": 0.09085705131292343 }, { "label": "completed_group_3", "mean": 0.07029052078723907, "median": 0.06780634075403214, "p05": 0.04846132919192314, "p95": 0.09917552024126053 }, { "label": "completed_group_4", "mean": 0.06935097277164459, "median": 0.06708167493343353, "p05": 0.04668639227747917, "p95": 0.09815650433301926 }, { "label": "completed_group_5", "mean": 0.056832700967788696, "median": 0.05595892667770386, "p05": 0.038822539150714874, "p95": 0.0771557092666626 }, { "label": "completed_group_6", "mean": 0.19425815343856812, "median": 0.19145764410495758, "p05": 0.12249615788459778, "p95": 0.27496030926704407 }, { "label": "completed_group_7", "mean": 0.17060703039169312, "median": 0.16317546367645264, "p05": 0.09239593893289566, "p95": 0.2726203501224518 }, { "label": "current_group_partial", "mean": 0.21711516380310059, "median": 0.21198657155036926, "p05": 0.11019522696733475, "p95": 0.3391069769859314 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 1.7837426662445068, "group": 8, "latest_source_mass": 0.29026859998703003, "layer": 30, "max_source_mass": 0.29026859998703003, "mixer_index": 59, "normalized_entropy": 0.8118162725118464, "offset": 2, "source_summaries": [ { "label": "embedding", "mean": 0.08836327493190765, "median": 0.07904411852359772, "p05": 0.02631576545536518, "p95": 0.17534467577934265 }, { "label": "completed_group_1", "mean": 0.04953558370471001, "median": 0.04658038914203644, "p05": 0.025181813165545464, "p95": 0.08318150043487549 }, { "label": "completed_group_2", "mean": 0.04813014343380928, "median": 0.045740917325019836, "p05": 0.026714935898780823, "p95": 0.07843323051929474 }, { "label": "completed_group_3", "mean": 0.0373263955116272, "median": 0.03401809185743332, "p05": 0.01990434154868126, "p95": 0.06290259957313538 }, { "label": "completed_group_4", "mean": 0.037015400826931, "median": 0.03405017405748367, "p05": 0.018646355718374252, "p95": 0.06544196605682373 }, { "label": "completed_group_5", "mean": 0.03390160948038101, "median": 0.03198609501123428, "p05": 0.018189728260040283, "p95": 0.05570102855563164 }, { "label": "completed_group_6", "mean": 0.1840541511774063, "median": 0.17327937483787537, "p05": 0.07966600358486176, "p95": 0.32982927560806274 }, { "label": "completed_group_7", "mean": 0.23140482604503632, "median": 0.22406929731369019, "p05": 0.08831030130386353, "p95": 0.40350067615509033 }, { "label": "current_group_partial", "mean": 0.29026859998703003, "median": 0.2745037376880646, "p05": 0.10280418395996094, "p95": 0.5399085879325867 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 1.9973840713500977, "group": 8, "latest_source_mass": 0.19101998209953308, "layer": 31, "max_source_mass": 0.20831164717674255, "mixer_index": 60, "normalized_entropy": 0.9090486661912383, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.10261696577072144, "median": 0.10389091819524765, "p05": 0.01371697522699833, "p95": 0.21163305640220642 }, { "label": "completed_group_1", "mean": 0.06039813905954361, "median": 0.06085003539919853, "p05": 0.026107093319296837, "p95": 0.09348798543214798 }, { "label": "completed_group_2", "mean": 0.06671924144029617, "median": 0.06656201928853989, "p05": 0.034684643149375916, "p95": 0.09819839894771576 }, { "label": "completed_group_3", "mean": 0.08377155661582947, "median": 0.0804532840847969, "p05": 0.05448959022760391, "p95": 0.12039553374052048 }, { "label": "completed_group_4", "mean": 0.07002593576908112, "median": 0.06737451255321503, "p05": 0.044841229915618896, "p95": 0.10655929893255234 }, { "label": "completed_group_5", "mean": 0.05768897384405136, "median": 0.056077782064676285, "p05": 0.039495617151260376, "p95": 0.08242568373680115 }, { "label": "completed_group_6", "mean": 0.20831164717674255, "median": 0.2021285891532898, "p05": 0.11652229726314545, "p95": 0.31706833839416504 }, { "label": "completed_group_7", "mean": 0.1594475358724594, "median": 0.1546279788017273, "p05": 0.06593266129493713, "p95": 0.2735573351383209 }, { "label": "current_group_partial", "mean": 0.19101998209953308, "median": 0.1713225245475769, "p05": 0.06408411264419556, "p95": 0.3850628137588501 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 1.969987154006958, "group": 8, "latest_source_mass": 0.16864418983459473, "layer": 31, "max_source_mass": 0.24503237009048462, "mixer_index": 61, "normalized_entropy": 0.8965797917640489, "offset": 3, "source_summaries": [ { "label": "embedding", "mean": 0.08120635151863098, "median": 0.061164144426584244, "p05": 0.0259944386780262, "p95": 0.23622018098831177 }, { "label": "completed_group_1", "mean": 0.07894632965326309, "median": 0.07658770680427551, "p05": 0.0496862456202507, "p95": 0.11596276611089706 }, { "label": "completed_group_2", "mean": 0.08615730702877045, "median": 0.08467075228691101, "p05": 0.050409119576215744, "p95": 0.12811316549777985 }, { "label": "completed_group_3", "mean": 0.07609779387712479, "median": 0.0722871869802475, "p05": 0.04099677503108978, "p95": 0.11950768530368805 }, { "label": "completed_group_4", "mean": 0.058728985488414764, "median": 0.056080836802721024, "p05": 0.03323862701654434, "p95": 0.09344407916069031 }, { "label": "completed_group_5", "mean": 0.05076971277594566, "median": 0.04946017265319824, "p05": 0.03047419898211956, "p95": 0.07563400268554688 }, { "label": "completed_group_6", "mean": 0.24503237009048462, "median": 0.24110376834869385, "p05": 0.10613486170768738, "p95": 0.3991503119468689 }, { "label": "completed_group_7", "mean": 0.1544169783592224, "median": 0.1400495171546936, "p05": 0.06199025735259056, "p95": 0.2995939254760742 }, { "label": "current_group_partial", "mean": 0.16864418983459473, "median": 0.15222924947738647, "p05": 0.0543956533074379, "p95": 0.3329208791255951 } ], "sources": 9 }, { "branch": "attention", "entropy_mean": 2.071648359298706, "group": 8, "latest_source_mass": 0.11520501971244812, "layer": 32, "max_source_mass": 0.20955611765384674, "mixer_index": 62, "normalized_entropy": 0.9428478002054053, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.10569609701633453, "median": 0.08689188957214355, "p05": 0.03173588588833809, "p95": 0.24571751058101654 }, { "label": "completed_group_1", "mean": 0.09922641515731812, "median": 0.09795533865690231, "p05": 0.06407676637172699, "p95": 0.14120742678642273 }, { "label": "completed_group_2", "mean": 0.08180510997772217, "median": 0.080217644572258, "p05": 0.054607540369033813, "p95": 0.11442367732524872 }, { "label": "completed_group_3", "mean": 0.10646799206733704, "median": 0.10510502755641937, "p05": 0.07415887713432312, "p95": 0.14216329157352448 }, { "label": "completed_group_4", "mean": 0.0826798751950264, "median": 0.08022409677505493, "p05": 0.05491747707128525, "p95": 0.11860857903957367 }, { "label": "completed_group_5", "mean": 0.06615597754716873, "median": 0.06485182791948318, "p05": 0.04477996751666069, "p95": 0.09238749742507935 }, { "label": "completed_group_6", "mean": 0.20955611765384674, "median": 0.206287682056427, "p05": 0.129981130361557, "p95": 0.2996031939983368 }, { "label": "completed_group_7", "mean": 0.13320739567279816, "median": 0.12465103715658188, "p05": 0.06303662061691284, "p95": 0.23065781593322754 }, { "label": "current_group_partial", "mean": 0.11520501971244812, "median": 0.10386130958795547, "p05": 0.03912060335278511, "p95": 0.22735844552516937 } ], "sources": 9 }, { "branch": "mlp", "entropy_mean": 1.9781701564788818, "group": 8, "latest_source_mass": 0.16112206876277924, "layer": 32, "max_source_mass": 0.22251944243907928, "mixer_index": 63, "normalized_entropy": 0.9003040366848136, "offset": 4, "source_summaries": [ { "label": "embedding", "mean": 0.09446848928928375, "median": 0.074150450527668, "p05": 0.042592622339725494, "p95": 0.19443637132644653 }, { "label": "completed_group_1", "mean": 0.07056106626987457, "median": 0.06868506222963333, "p05": 0.04818308353424072, "p95": 0.0982733890414238 }, { "label": "completed_group_2", "mean": 0.08033448457717896, "median": 0.08047192543745041, "p05": 0.048296794295310974, "p95": 0.11253741383552551 }, { "label": "completed_group_3", "mean": 0.0658978745341301, "median": 0.06256154924631119, "p05": 0.04223886504769325, "p95": 0.0967884361743927 }, { "label": "completed_group_4", "mean": 0.058234501630067825, "median": 0.0563921295106411, "p05": 0.03529864549636841, "p95": 0.08846980333328247 }, { "label": "completed_group_5", "mean": 0.04316519945859909, "median": 0.042268507182598114, "p05": 0.029443470761179924, "p95": 0.05996086448431015 }, { "label": "completed_group_6", "mean": 0.2036968618631363, "median": 0.19645483791828156, "p05": 0.09398645162582397, "p95": 0.3395216464996338 }, { "label": "completed_group_7", "mean": 0.22251944243907928, "median": 0.2130982130765915, "p05": 0.11867167800664902, "p95": 0.35465875267982483 }, { "label": "current_group_partial", "mean": 0.16112206876277924, "median": 0.14023703336715698, "p05": 0.06008770316839218, "p95": 0.3247341811656952 } ], "sources": 9 }, { "branch": "output", "entropy_mean": 1.6036605834960938, "group": null, "latest_source_mass": 0.31639498472213745, "layer": null, "max_source_mass": 0.3412485122680664, "mixer_index": null, "normalized_entropy": 0.7298573846467136, "offset": null, "source_summaries": [ { "label": "embedding", "mean": 0.09759747982025146, "median": 0.09915056824684143, "p05": 0.025461740791797638, "p95": 0.16398483514785767 }, { "label": "completed_group_1", "mean": 0.04215940088033676, "median": 0.04050712287425995, "p05": 0.025720715522766113, "p95": 0.06398092210292816 }, { "label": "completed_group_2", "mean": 0.041145481169223785, "median": 0.03881259262561798, "p05": 0.023510465398430824, "p95": 0.0662468671798706 }, { "label": "completed_group_3", "mean": 0.029554028064012527, "median": 0.027420252561569214, "p05": 0.01627083122730255, "p95": 0.04924973472952843 }, { "label": "completed_group_4", "mean": 0.0289996936917305, "median": 0.027226522564888, "p05": 0.01560397818684578, "p95": 0.048351090401411057 }, { "label": "completed_group_5", "mean": 0.03647579252719879, "median": 0.0347352996468544, "p05": 0.020744025707244873, "p95": 0.058257561177015305 }, { "label": "completed_group_6", "mean": 0.06642461568117142, "median": 0.057809896767139435, "p05": 0.022837381809949875, "p95": 0.14078694581985474 }, { "label": "completed_group_7", "mean": 0.3412485122680664, "median": 0.3336990475654602, "p05": 0.0819513276219368, "p95": 0.6423859596252441 }, { "label": "completed_group_8", "mean": 0.31639498472213745, "median": 0.30713334679603577, "p05": 0.05841828137636185, "p95": 0.6156357526779175 } ], "sources": 9 } ], "mode": "learned", "optimizer_state_after": "ed03e6fbd4a12d8b063dcb22e0437754285f54d585374cd52fbd534f05d24637", "optimizer_state_before": "ed03e6fbd4a12d8b063dcb22e0437754285f54d585374cd52fbd534f05d24637", "optimizer_state_unchanged": true, "positions": { "attention_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "5e6ecbcd78543fc42d4098a51169dca446d404ce384015643e84d2c78433c949", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.0933690071105957, 0.24537226557731628, 0.07698935270309448, 0.0523640513420105, 0.09380833059549332, 0.03647654876112938, 0.039120711386203766, 0.06371714919805527, 0.05156813934445381, 0.03728010877966881, 0.03983496502041817, 0.040284499526023865, 0.046871740370988846, 0.04314620420336723, 0.04033593460917473, 0.04471700266003609, 0.08397205173969269, 0.0528092160820961, 0.04443390294909477, 0.07560808956623077, 0.03708760067820549, 0.03599191829562187, 0.033304885029792786, 0.04061548039317131, 0.025487789884209633, 0.021565141156315804, 0.02342291921377182, 0.017244853079319, 0.024166326969861984, 0.016819356009364128, 0.021590016782283783, 0.026201345026493073 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.6127158914969186e-05, "first_to_last_ratio": 0.6403133863364645, "imbalance_abs_log_ratio": 0.4457975563250414, "last_quartile_mean": 2.518635290016391e-05, "mean": 2.1593947650444534e-05, "normalized": [ 0.6929919556166857, 0.8032314127318161, 0.7123629256452019, 0.6640523991876173, 0.8725736234454132, 0.8071370183700546, 0.7516577913014878, 0.6706888120378312, 0.5882487760705537, 0.5190639183458972, 0.4863138588077732, 0.4652918617004374, 0.6603036411218667, 0.6189718629251246, 0.5802646281311779, 0.5410038780854491, 0.6697462103535458, 0.6259079034957141, 0.6246747715872017, 0.6676260304525075, 3.1377243999857436, 2.6752853614525707, 2.136977079899559, 1.6970064982433113, 2.2713553531497497, 1.9850638882393974, 1.4820693911808296, 0.9615863897898643, 1.0163024380148542, 0.7191699586662397, 0.4720839007000451, 0.42326206126447913 ], "peak_layer": 21, "peak_normalized": 3.1377243999857436, "population_cv": 0.6918392109604528, "rest_mean": 1.6060819520875897e-05, "spike_contrast": 3.2048700555541787, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.147283955011517e-05, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 1.4964432011765894e-05, 1.7344937077723444e-05, 1.53827277245e-05, 1.4339512745209504e-05, 1.8842309145838954e-05, 1.7429274521418847e-05, 1.623125899641309e-05, 1.4482819096883759e-05, 1.2702613275905605e-05, 1.1208639079995919e-05, 1.0501436008780729e-05, 1.0047488103737123e-05, 1.4258562259783503e-05, 1.3366046005103271e-05, 1.2530204003269318e-05, 1.1682409422064666e-05, 1.446246460545808e-05, 1.3515822502085939e-05, 1.3489194316207431e-05, 1.4416681551665533e-05, 6.775585643481463e-05, 5.7769972045207396e-05, 4.61457711935509e-05, 3.664506948553026e-05, 4.9047528591472656e-05, 4.2865365685429424e-05, 3.2003728847485036e-05, 2.076444616250228e-05, 2.1945981643511914e-05, 1.5529718439211138e-05, 1.019415503833443e-05, 9.13989879336441e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 6.422064188882359e-05, "first_to_last_ratio": 0.6376814356469364, "imbalance_abs_log_ratio": 0.4499164375054873, "last_quartile_mean": 0.00010070959933727863, "mean": 8.629182207187114e-05, "normalized": [ 0.6940773401510585, 0.8062528427202595, 0.7180374505283312, 0.6647119372719499, 0.8588689882759897, 0.7941431696191276, 0.7473948010342778, 0.6703237246625169, 0.591781846497542, 0.522694550559361, 0.4893812082436792, 0.4670959071660659, 0.6619658672285874, 0.6164719403371215, 0.5787012511245688, 0.5393739243504363, 0.6707710740427385, 0.62668399370843, 0.6317390712162502, 0.6712190136492653, 3.138964807406113, 2.6795820619127526, 2.136435916928567, 1.6866743661161174, 2.264472697470994, 1.978410075097681, 1.4778942897678682, 0.9605324338748193, 1.022040338716437, 0.7239267286003249, 0.47869752189559756, 0.4306788598251707 ], "peak_layer": 21, "peak_normalized": 3.138964807406113, "population_cv": 0.6905024564046187, "rest_mean": 6.421987658271703e-05, "spike_contrast": 3.1996375366532934, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00020548032771330326, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 5.989319834043272e-05, 6.957302684895694e-05, 6.196075992193073e-05, 5.7359204220119864e-05, 7.411336991935968e-05, 6.852806109236553e-05, 6.449405918829143e-05, 5.784345557913184e-05, 5.1065933803329244e-05, 4.5104265154805034e-05, 4.222959614708088e-05, 4.03065569116734e-05, 5.712224083254114e-05, 5.319648698787205e-05, 4.993718539481051e-05, 4.654355871025473e-05, 5.788205817225389e-05, 5.4077703680377454e-05, 5.4513915529241785e-05, 5.792071169707924e-05, 0.0002708669926505536, 0.00023122601851355284, 0.00018435694801155478, 0.00014554620429407805, 0.00019540547509677708, 0.0001707206101855263, 0.00012753019109368324, 8.288609387818724e-05, 8.81937230587937e-05, 6.246895645745099e-05, 4.1307681385660544e-05, 3.716406354214996e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.05695150885730982, "first_to_last_ratio": 0.6376814663645546, "imbalance_abs_log_ratio": 0.4499163893346989, "last_quartile_mean": 0.08931027771905065, "mean": 0.07652445149142295, "normalized": [ 0.6940773558515339, 0.8062528453975966, 0.7180374361894372, 0.6647119756359035, 0.8588690718004696, 0.7941431432641803, 0.7473947840325739, 0.6703237318272384, 0.5917818305157142, 0.5226945671206367, 0.4893812028028422, 0.46709594515141906, 0.6619658291829611, 0.6164719318106706, 0.5787012336125282, 0.5393739103061749, 0.6707710619535627, 0.6266839981127393, 0.6317390378687008, 0.6712190249334776, 3.138964907679676, 2.679582133407049, 2.136436026253706, 1.6866743790728473, 2.2644726279739604, 1.9784099533155926, 1.4778942270811004, 0.9605324250805215, 1.0220403417906543, 0.7239267230206866, 0.4786975145327867, 0.43067882342105884 ], "peak_layer": 21, "peak_normalized": 3.138964907679676, "population_cv": 0.6905024682030625, "rest_mean": 0.05695082868138949, "spike_contrast": 3.1996376327558256, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.18222201466560364, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 0.05311388894915581, 0.061698056757450104, 0.054947420954704285, 0.05086671933531761, 0.06572448462247849, 0.06077136844396591, 0.057193975895643234, 0.05129615589976311, 0.04528577998280525, 0.039998915046453476, 0.03744962811470032, 0.035744260996580124, 0.05065657198429108, 0.047175176441669464, 0.044284794479608536, 0.04127529263496399, 0.051330387592315674, 0.04795664921402931, 0.0483434833586216, 0.05136466771364212, 0.24020756781101227, 0.20505355298519135, 0.16348959505558014, 0.12907183170318604, 0.1732875257730484, 0.1513967365026474, 0.11309504508972168, 0.07350421696901321, 0.07821107655763626, 0.05539809539914131, 0.03663206472992897, 0.03295746073126793 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 2.592033183645981e-06, "first_to_last_ratio": 0.3776007228014791, "imbalance_abs_log_ratio": 0.9739179305137912, "last_quartile_mean": 6.864481520096888e-06, "mean": 4.875440396290287e-06, "normalized": [ 0.58913988046903, 0.5446521181917551, 0.5278862817384906, 0.53898932284961, 0.5219441801529001, 0.509889287242028, 0.5048315077449682, 0.5158761132703409, 0.4368296481828547, 0.4047199700448834, 0.3956716882892422, 0.40126275663148436, 0.5110508169296636, 0.5085656955747676, 0.5112234187675316, 0.511163584085558, 0.5662253891905324, 0.5872486754624854, 0.609842074156892, 0.6494088419115644, 3.025689231381945, 2.8686186674565652, 2.4420116557657345, 2.053486551003887, 2.698068277749679, 2.429082514259166, 1.8298867229318267, 1.2191127574146867, 1.182073363720965, 0.8592876866491284, 0.5581944369380665, 0.48806688384176716 ], "peak_layer": 21, "peak_normalized": 3.025689231381945, "population_cv": 0.8235639638153561, "rest_mean": 3.4149978226330994e-06, "spike_contrast": 3.736995148125518, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.2761830294039101e-05, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 2.87231637230434e-06, 2.6554189389571548e-06, 2.573678102635313e-06, 2.6278103177901357e-06, 2.5447077405260643e-06, 2.485934828655445e-06, 2.461275926179951e-06, 2.5151232421194436e-06, 2.1297369130479638e-06, 1.9731880911422195e-06, 1.92907373275375e-06, 1.9563326532079373e-06, 2.491597797416034e-06, 2.4794817363726906e-06, 2.4924393073888496e-06, 2.492147586963256e-06, 2.7605981358647114e-06, 2.863095915017766e-06, 2.973248683701968e-06, 3.166154101563734e-06, 1.4751567505300045e-05, 1.3985779332870152e-05, 1.1905882274731994e-05, 1.0011651284003165e-05, 1.3154271073290147e-05, 1.1842847015941516e-05, 8.92150364961708e-06, 5.943711585132405e-06, 5.763128228863934e-06, 4.18940589952399e-06, 2.7214437068323605e-06, 2.3795410015736707e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 4.8627718115312746e-05, "first_to_last_ratio": 0.778294860341054, "imbalance_abs_log_ratio": 0.2506498287376175, "last_quartile_mean": 6.247981400520075e-05, "mean": 6.0048676687074476e-05, "normalized": [ 0.768607661628769, 0.8690595287737466, 0.7879569251847977, 0.7385864958726717, 0.9274786983184008, 0.8617431831538573, 0.8054883103443636, 0.7195191288766225, 0.6496482220749353, 0.5799654886485536, 0.5429742744436054, 0.5176928240267351, 0.720980532018643, 0.6651070950014618, 0.6225467122437304, 0.5784265987882409, 0.7443177261701239, 0.697888823545442, 0.701818835999622, 0.7379727210931422, 3.1748914800668615, 2.647277084571918, 2.0581907003495648, 1.5579720703470805, 2.1379505239352077, 1.7650981274587072, 1.310550617537982, 0.8840683850126465, 0.8342765345805514, 0.5872947992080164, 0.42062138312406294, 0.3840285075999366 ], "peak_layer": 21, "peak_normalized": 3.1748914800668615, "population_cv": 0.6535432926484233, "rest_mean": 4.5422861014096134e-05, "spike_contrast": 3.0607513093024403, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013902808132115752, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 4.615387297235429e-05, 5.218587466515601e-05, 4.7315770643763244e-05, 4.435114169609733e-05, 5.569386848947033e-05, 5.174653779249638e-05, 4.83685071230866e-05, 4.3206171540077776e-05, 3.901051604771055e-05, 3.482616011751816e-05, 3.2604886655462906e-05, 3.1086769013199955e-05, 4.329392686486244e-05, 3.993880091002211e-05, 3.738310624612495e-05, 3.473375181783922e-05, 4.4695294491248205e-05, 4.1907300328603014e-05, 4.214329237584025e-05, 4.431428533280268e-05, 0.00019064803200308233, 0.0001589654857525602, 0.00012359162792563438, 9.355416113976389e-05, 0.00012838109978474677, 0.00010599180677672848, 7.869683031458408e-05, 5.3087136620888487e-05, 5.009720189264044e-05, 3.52662755176425e-05, 2.525775744288694e-05, 2.3060403691488318e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 3.645518927442026e-05, "first_to_last_ratio": 0.9865830246987611, "imbalance_abs_log_ratio": 0.013507796190281324, "last_quartile_mean": 3.6950959383830195e-05, "mean": 4.1335310356771515e-05, "normalized": [ 0.8560051014711698, 0.9467981965967722, 0.8689980494059457, 0.8159268191754907, 1.0103083717536356, 0.9426554996877177, 0.8489560918930947, 0.765857482001633, 0.7201445334320749, 0.6498760503731635, 0.6109444686941845, 0.5791405787907885, 0.7989198470081217, 0.7291371869136885, 0.6790968935040115, 0.6275849613644364, 0.8454614229708973, 0.7893993011096712, 0.7856135784658201, 0.8181223517247423, 3.2763150867214206, 2.615409622894913, 1.9276349807221327, 1.3402369957405367, 1.940332562376331, 1.4667183642001942, 1.0945246384976166, 0.8103162680270223, 0.6598082696260246, 0.4694457579306494, 0.3666062775483438, 0.3437043893777548 ], "peak_layer": 21, "peak_normalized": 3.2763150867214206, "population_cv": 0.6207682663855968, "rest_mean": 3.199670040716969e-05, "spike_contrast": 2.867914594845514, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.176380408462136e-05, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 3.5383236536290497e-05, 3.913619730155915e-05, 3.592030407162383e-05, 3.37265882990323e-05, 4.176141010248102e-05, 3.896495763910934e-05, 3.509186353767291e-05, 3.1656956707593054e-05, 2.9767397791147232e-05, 2.6862828235607594e-05, 2.5253579224226996e-05, 2.3938955564517528e-05, 3.3023599826265126e-05, 3.0139111913740635e-05, 2.8070680855307728e-05, 2.594141915324144e-05, 3.4947410313179716e-05, 3.263006510678679e-05, 3.2473581086378545e-05, 3.381734131835401e-05, 0.0001354275009362027, 0.00010810876847244799, 7.967939018271863e-05, 5.539911217056215e-05, 8.020424866117537e-05, 6.062725879019126e-05, 4.5242515625432134e-05, 3.349467442603782e-05, 2.7273379600956105e-05, 1.9404686099733226e-05, 1.5153784261201508e-05, 1.4207127605914138e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00012982768839719938, "first_to_last_ratio": 0.6037915882942751, "imbalance_abs_log_ratio": 0.5045261930845532, "last_quartile_mean": 0.00021502069739653962, "mean": 0.0001790727353636612, "normalized": [ 0.6739765257680471, 0.7636067305383979, 0.68849644503889, 0.6447045606228033, 0.8477445590156324, 0.7818709864142286, 0.7391626168129651, 0.6604355769831363, 0.5800901925673246, 0.5130947085748045, 0.4841531977734674, 0.46136480942786895, 0.6373637447050293, 0.5885397553357715, 0.5617314689244236, 0.5244758601010534, 0.6356987553893274, 0.6101287890369049, 0.6118132813754859, 0.6643470301154188, 3.09815074914609, 2.693324715468958, 2.18351992306606, 1.7462447599227975, 2.367437166894702, 2.0445672235354038, 1.5627333414952298, 1.026078331181794, 1.036030398706582, 0.6947075889876938, 0.46018158540857673, 0.4142246216651318 ], "peak_layer": 21, "peak_normalized": 3.09815074914609, "population_cv": 0.7095514290759102, "rest_mean": 0.00013205833400004648, "spike_contrast": 3.278479211520481, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0004329505027271807, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 0.00012069082004018128, 0.00013674114597961307, 0.00012329094170127064, 0.00011544900917215273, 0.00015180793707259, 0.00014001177623867989, 0.0001323638716712594, 0.00011826600530184805, 0.00010387833754066378, 9.188127296511084e-05, 8.669863746035844e-05, 8.261785842478275e-05, 0.00011413446918595582, 0.00010539142385823652, 0.00010059079068014398, 9.391932690050453e-05, 0.00011383631499484181, 0.00010925743117695674, 0.00010955907782772556, 0.00011896643991349265, 0.0005547943292185664, 0.0004823010240215808, 0.0003910088853444904, 0.0003127048257738352, 0.00042394344927743077, 0.0003661262453533709, 0.0002798429341055453, 0.0001837426534621045, 0.00018552479741629213, 0.0001244031882379204, 8.240597526310012e-05, 7.417633605655283e-05 ] } } }, "mlp_branch_output": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "c0b8018c349b1bd09e5d17858eb312a2323da0e92de9651e2bb08ae4adf115c7", "count": 32, "dtypes": [ "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16", "torch.bfloat16" ], "output_rms_by_layer": [ 0.13583384454250336, 0.0415157787501812, 0.018140379339456558, 0.039518751204013824, 0.022877270355820656, 0.015532293356955051, 0.02314678393304348, 0.05432315170764923, 0.03743875026702881, 0.04006272554397583, 0.045583780854940414, 0.05995568633079529, 0.038651175796985626, 0.04559725895524025, 0.05632574483752251, 0.07952121645212173, 0.042337141931056976, 0.05044656619429588, 0.04418695345520973, 0.06310147047042847, 0.029010163620114326, 0.04586305841803551, 0.06033914536237717, 0.07656749337911606, 0.045884523540735245, 0.051604680716991425, 0.06502768397331238, 0.06792232394218445, 0.060306549072265625, 0.08025261014699936, 0.09663844108581543, 0.12230555713176727 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.5986858215910615e-05, "first_to_last_ratio": 0.8170320074925627, "imbalance_abs_log_ratio": 0.20207700803204537, "last_quartile_mean": 1.9566991340980167e-05, "mean": 1.9039267499465495e-05, "normalized": [ 0.7853354674160987, 0.9097466615865485, 0.8087937381555731, 0.7523818308832718, 0.9858416226009721, 0.9045115156777315, 0.8288509697382657, 0.741963942115696, 0.6090758738841542, 0.5628736284922787, 0.5329013290216021, 0.5131548852635935, 0.7265824288035061, 0.6757039396775449, 0.6225875966900762, 0.5898833165121294, 0.7532717273459274, 0.6927780667296713, 0.6897910451069404, 0.809449516707258, 3.207633782072864, 2.5002059817245676, 1.9824571594317655, 1.5924827370882133, 2.273874867822024, 1.6940739554643818, 1.0968135124011316, 0.8719494928225496, 0.815651324531277, 0.538892521487639, 0.48196570767333513, 0.44851985507141146 ], "peak_layer": 21, "peak_normalized": 3.207633782072864, "population_cv": 0.6428607369119902, "rest_mean": 1.4415789741582961e-05, "spike_contrast": 3.0526282764165082, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.4006047392031175e-05, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 1.4952212040952872e-05, 1.7320910046692006e-05, 1.539884033263661e-05, 1.4324798939924221e-05, 1.8769702364807017e-05, 1.7221236703335308e-05, 1.578071533003822e-05, 1.4126449968898669e-05, 1.1596358490351122e-05, 1.0716701581259258e-05, 1.0146050954062957e-05, 9.770093129191082e-06, 1.3833597222401295e-05, 1.2864908057963476e-05, 1.18536117952317e-05, 1.1230946256546304e-05, 1.434174191672355e-05, 1.318998693022877e-05, 1.3133116226526909e-05, 1.541132587590255e-05, 6.107099761720747e-05, 4.760209048981778e-05, 3.77445321646519e-05, 3.0319704819703475e-05, 4.3292911868775263e-05, 3.225392720196396e-05, 2.088252585963346e-05, 1.660127963987179e-05, 1.5529403754044324e-05, 1.0260118870064616e-05, 9.176274033961818e-06, 8.539489499526098e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 6.365280069076107e-05, "first_to_last_ratio": 0.8123684077728273, "imbalance_abs_log_ratio": 0.2078013375593398, "last_quartile_mean": 7.83545988269907e-05, "mean": 7.614439766712167e-05, "normalized": [ 0.7854660713099145, 0.9130496523819581, 0.8147298517264512, 0.752236196131495, 0.9689924576782674, 0.8892579571020611, 0.8242433765322099, 0.7396130336387524, 0.6125904066287632, 0.5670744382794471, 0.5359292329408729, 0.5145351492524995, 0.7290151059417195, 0.673146260009013, 0.621425875046763, 0.5883337897938153, 0.7535116607120764, 0.6934400787203782, 0.6978162466203766, 0.8168476312813977, 3.2190719364475022, 2.5076839554965473, 1.973830744692153, 1.575947332799018, 2.2647184639164535, 1.6880790923847124, 1.0924366554795215, 0.8732736568641918, 0.8206255787733402, 0.5459821164389519, 0.48994786526283923, 0.4571481297165359 ], "peak_layer": 21, "peak_normalized": 3.2190719364475022, "population_cv": 0.642032487569562, "rest_mean": 5.76970003883751e-05, "spike_contrast": 3.0462648281411457, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001757603429723531, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 5.9808840887853876e-05, 6.952361582079902e-05, 6.203711382113397e-05, 5.727857205783948e-05, 7.378334703389555e-05, 6.771201151423156e-05, 6.276151543715969e-05, 5.6317388953175396e-05, 4.664532752940431e-05, 4.3179541535209864e-05, 4.0808008634485304e-05, 3.917896901839413e-05, 5.551041613216512e-05, 5.125631651026197e-05, 4.731809895019978e-05, 4.479832205106504e-05, 5.7375691540073603e-05, 5.280157711240463e-05, 5.3134797781240195e-05, 6.219837086973712e-05, 0.00024511429364793, 0.00019094608433078974, 0.0001502961531514302, 0.00011999956041108817, 0.00017244562332052737, 0.00012853776570409536, 8.318293112097308e-05, 6.649489660048857e-05, 6.248604040592909e-05, 4.157347939326428e-05, 3.7306785088730976e-05, 3.480926898191683e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.05644794087857008, "first_to_last_ratio": 0.8123684042388034, "imbalance_abs_log_ratio": 0.2078013419096122, "last_quartile_mean": 0.06948564294725657, "mean": 0.06752561370376498, "normalized": [ 0.7854660588804813, 0.9130496786513765, 0.814729886022376, 0.7522361645698925, 0.968992465622299, 0.8892579649660906, 0.8242433717205022, 0.7396130482968516, 0.6125904375263521, 0.5670744516239514, 0.535929266538159, 0.5145351229333333, 0.7290151143815181, 0.673146253269631, 0.6214258465577596, 0.5883338184574164, 0.7535116614810096, 0.6934400601144122, 0.6978162502950979, 0.8168476412095322, 3.21907174067581, 2.5076838612014067, 1.9738308079311102, 1.5759473804424433, 2.264718470873259, 1.6880792267982687, 1.0924366234670762, 0.8732736495232089, 0.8206255836175469, 0.5459820799959341, 0.4899478787351448, 0.45714813362074885 ], "peak_layer": 21, "peak_normalized": 3.21907174067581, "population_cv": 0.6420324679566953, "rest_mean": 0.05116627765474496, "spike_contrast": 3.0462647570380494, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.15586602836847305, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 0.053039077669382095, 0.061654239892959595, 0.05501513555645943, 0.05079520866274834, 0.06543181091547012, 0.0600476898252964, 0.05565753951668739, 0.049942824989557266, 0.041365545243024826, 0.0382920503616333, 0.03618895262479782, 0.034744299948215485, 0.049227192997932434, 0.045454613864421844, 0.0419621616601944, 0.039727602154016495, 0.050881337374448776, 0.04682496562600136, 0.04712047055363655, 0.055158138275146484, 0.21736979484558105, 0.16933289170265198, 0.13328413665294647, 0.10641681402921677, 0.152926504611969, 0.11398858577013016, 0.07376745343208313, 0.05896833911538124, 0.055413246154785156, 0.03686777502298355, 0.03308403119444847, 0.03086920827627182 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 2.57929028180115e-06, "first_to_last_ratio": 0.4759084167214945, "imbalance_abs_log_ratio": 0.7425298451079264, "last_quartile_mean": 5.41971982670475e-06, "mean": 4.213722270662856e-06, "normalized": [ 0.6771451422985035, 0.6338625145334242, 0.6120949335724296, 0.6258725064817227, 0.6050121598754559, 0.5852465596314514, 0.5785033580390577, 0.5791973952882404, 0.4696864977471604, 0.4522091645384124, 0.4457992216966758, 0.45590579514105095, 0.5773211960070957, 0.5667516162949074, 0.5644132471304436, 0.568160519465189, 0.6503432281671621, 0.663170344529487, 0.6837490184222774, 0.7583004470800568, 3.021070651777523, 2.6274098317695387, 2.3058345630897463, 2.003282670865861, 2.819487095674051, 2.140274896548892, 1.4825743986898414, 1.122684410298403, 0.9855482185618173, 0.6429507763662341, 0.5708772581826982, 0.5252603622351902 ], "peak_layer": 21, "peak_normalized": 3.021070651777523, "population_cv": 0.7577432386283482, "rest_mean": 3.000000956658537e-06, "spike_contrast": 3.5892713108597136, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.0767817366286181e-05, "top_five_layers": [ 21, 25, 22, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 2.8533015665743733e-06, 2.670920594027848e-06, 2.579198053354048e-06, 2.6372529191576177e-06, 2.549353212089045e-06, 2.4660664621478645e-06, 2.4376524834224256e-06, 2.4405769636359764e-06, 1.9791284557868494e-06, 1.9054838276133523e-06, 1.8784741087074508e-06, 1.921060402310104e-06, 2.4326711809408152e-06, 2.3881339075160213e-06, 2.378280669290689e-06, 2.394070634181844e-06, 2.7403657441027462e-06, 2.7944156499870587e-06, 2.881128466469818e-06, 3.1952674817148363e-06, 1.27299526866409e-05, 1.1071175322285853e-05, 9.71614645095542e-06, 8.441276804660447e-06, 1.1880535566888284e-05, 9.018523996928707e-06, 6.247156761673978e-06, 4.730680302600376e-06, 4.152826477366034e-06, 2.7092160053143743e-06, 2.4055182166193845e-06, 2.2133012862468604e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 4.826902750210138e-05, "first_to_last_ratio": 0.9723604213825404, "imbalance_abs_log_ratio": 0.028028739360968564, "last_quartile_mean": 4.964108620697516e-05, "mean": 5.395586174472555e-05, "normalized": [ 0.8546738942457572, 0.9667007478721396, 0.8780654782701063, 0.82151592085777, 1.0287569123437954, 0.9492731152891766, 0.8750422046820956, 0.7827885676521403, 0.6672237173045179, 0.6202571530123847, 0.5854234638750982, 0.5620770546607131, 0.7815896821357725, 0.715521927114241, 0.659703864497871, 0.6215694475292887, 0.8222045332968153, 0.7616919988241317, 0.764120987673305, 0.8753289635877315, 3.236716069528407, 2.479479992593944, 1.8829977352397056, 1.4470254883420661, 2.056662650357765, 1.5065673712263334, 1.0009483781315074, 0.8155586445002443, 0.6595748803016609, 0.47597993512775083, 0.43187920683965, 0.4130800130861142 ], "peak_layer": 21, "peak_normalized": 3.236716069528407, "population_cv": 0.610766278714797, "rest_mean": 4.176007456003895e-05, "spike_contrast": 2.8690828214345374, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0001198131125420332, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 4.6114666474750265e-05, 5.2159171900711954e-05, 4.737677954835817e-05, 4.432559944689274e-05, 5.550746573135257e-05, 5.121884896652773e-05, 4.721365621662699e-05, 4.223603173159063e-05, 3.600063064368442e-05, 3.3466509194113314e-05, 3.158702747896314e-05, 3.032735185115598e-05, 4.217134483042173e-05, 3.860660217469558e-05, 3.5594890505308285e-05, 3.353731517563574e-05, 4.4362754124449566e-05, 4.109774818061851e-05, 4.122880636714399e-05, 4.722912854049355e-05, 0.00017463980475440621, 0.00013378247967921197, 0.00010159876546822488, 7.80755071900785e-05, 0.00011096900561824441, 8.128814079100266e-05, 5.400703230407089e-05, 4.400416946737096e-05, 3.558793105185032e-05, 2.568190757301636e-05, 2.3302414774661884e-05, 2.22880880755838e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 3.626883517426904e-05, "first_to_last_ratio": 1.1972713068748915, "imbalance_abs_log_ratio": 0.1800450565949769, "last_quartile_mean": 3.0292912697404972e-05, "mean": 3.794855416572318e-05, "normalized": [ 0.934241229967689, 1.0326674049103126, 0.9483113974161322, 0.8884949951731808, 1.1009339220259078, 1.0148817561286225, 0.9027362835592571, 0.8236285600155873, 0.7333418938739371, 0.6823440677494514, 0.6463725772705814, 0.6166567793546124, 0.850307320786337, 0.7701976529963521, 0.7042915982263429, 0.6605966097637871, 0.9142336932467707, 0.8480352948917127, 0.839474071242197, 0.9443144532969522, 3.2998505290664992, 2.4485943259044243, 1.7642227386757172, 1.2451697951023746, 1.763789999228952, 1.2716404306318911, 0.9094502638769789, 0.7804639985266583, 0.5161011792987255, 0.40746526774101827, 0.37633786588868356, 0.36085204416235356 ], "peak_layer": 21, "peak_normalized": 3.2998505290664992, "population_cv": 0.5857907121700514, "rest_mean": 3.01878957947742e-05, "spike_contrast": 2.6453022731935993, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 7.985610936884768e-05, "top_five_layers": [ 21, 22, 23, 25, 26 ], "top_five_spike_overlap": 4 }, "values": [ 3.545310391928069e-05, 3.918823495041579e-05, 3.5987046430818737e-05, 3.3717100450303406e-05, 4.177885057288222e-05, 3.851329529425129e-05, 3.425753675401211e-05, 3.12555130221881e-05, 2.7829264581669122e-05, 2.5893970814649947e-05, 2.452890475979075e-05, 2.3401233192998916e-05, 3.2267933420371264e-05, 2.9227887353044935e-05, 2.672684786375612e-05, 2.5068686227314174e-05, 3.469384682830423e-05, 3.218171332264319e-05, 3.185682726325467e-05, 3.583536818041466e-05, 0.00012522455654107034, 9.292061440646648e-05, 6.694970215903595e-05, 4.72523934149649e-05, 6.693328032270074e-05, 4.825691576115787e-05, 3.4512322599766776e-05, 2.961748032248579e-05, 1.9585293557611294e-05, 1.546271778352093e-05, 1.4281477888289373e-05, 1.3693813343707006e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00012846768913732376, "first_to_last_ratio": 0.7741677488258334, "imbalance_abs_log_ratio": 0.2559666991257704, "last_quartile_mean": 0.00016594296175753698, "mean": 0.0001574651118971815, "normalized": [ 0.7618335990450072, 0.8698560675479529, 0.7822046486498666, 0.7351395196601278, 0.9594102821252986, 0.8815927774487453, 0.812858049389013, 0.7238937569203876, 0.5978223149190788, 0.5606129559633132, 0.5319350341866579, 0.513003926829316, 0.6981154037320007, 0.6408130676433959, 0.6075107239537738, 0.5706379352682298, 0.7181374534396903, 0.6783796513052409, 0.6819318446184696, 0.7908903240994326, 3.2180960304949577, 2.5452212715171636, 2.028532306848687, 1.6608547003669625, 2.321647122642583, 1.7738587692821628, 1.1712480907534195, 0.9285924776215382, 0.7896984661161683, 0.5235677697382035, 0.4733567899608167, 0.4487468679123384 ], "peak_layer": 21, "peak_normalized": 3.2180960304949577, "population_cv": 0.6626251267593958, "rest_mean": 0.00011795681536939271, "spike_contrast": 3.1436073616095452, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00037080991314724087, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 0.00011996221292065457, 0.00013697198301088065, 0.00012316994252614677, 0.00011575882672332227, 0.0001510736474301666, 0.0001388201053487137, 0.0001279967837035656, 0.00011398801143513992, 9.413615771336481e-05, 8.82769818417728e-05, 8.376120968023315e-05, 8.078022074187174e-05, 0.00010992882016580552, 0.00010090570140164346, 9.566174412611872e-05, 8.985556632978842e-05, 0.0001130815944634378, 0.00010682112770155072, 0.00010738047421909869, 0.00012453763338271528, 0.0005067378515377641, 0.0004007835523225367, 0.0003194230666849762, 0.0002615266712382436, 0.0003655784239526838, 0.0002793208695948124, 0.00018443071166984737, 0.0001462209183955565, 0.00012434995733201504, 8.244365744758397e-05, 7.453717989847064e-05, 7.066197576932609e-05 ] } } }, "post_attention_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "90a633f12d984935028fe5d7212be48e7da7b4a0806751939a551938d16dc540", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.0933690071105957, 0.35791492462158203, 0.3940892517566681, 0.4057151973247528, 0.09380833059549332, 0.11286329478025436, 0.13231655955314636, 0.1621972620487213, 0.05156813934445381, 0.08533531427383423, 0.12003090232610703, 0.16063299775123596, 0.046871740370988846, 0.079384446144104, 0.11459959298372269, 0.15625107288360596, 0.08397205173969269, 0.11654024571180344, 0.15294702351093292, 0.21572710573673248, 0.03708760067820549, 0.06553087383508682, 0.09390194714069366, 0.12852425873279572, 0.025487789884209633, 0.06059965491294861, 0.08860157430171967, 0.12100546807050705, 0.024166326969861984, 0.06877554953098297, 0.12682415544986725, 0.2025238573551178 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.6127134472299076e-05, "first_to_last_ratio": 0.6403082943802734, "imbalance_abs_log_ratio": 0.4458055086442437, "last_quartile_mean": 2.5186515017594502e-05, "mean": 2.159394537670778e-05, "normalized": [ 0.6929830574423667, 0.8032565154251953, 0.7123500282965459, 0.6640666207712503, 0.8725494553308608, 0.8071350816916083, 0.7516540798235798, 0.6706926732818747, 0.5882490907185235, 0.5190611932100045, 0.48631129869562834, 0.46529523801872363, 0.6602888008616835, 0.6189890701429912, 0.5802627939183058, 0.5409897412703016, 0.6697392892795451, 0.6259125602673482, 0.6246778698614331, 0.6676379780381497, 3.137658015393984, 2.675256160517513, 2.137000554072058, 1.6970284098395438, 2.2713857488305376, 1.9850977916906365, 1.4820653354312663, 0.9615821107638199, 1.0163133272453195, 0.7191715506408339, 0.4720786856528992, 0.423259873575668 ], "peak_layer": 21, "peak_normalized": 3.137658015393984, "population_cv": 0.691835904675886, "rest_mean": 1.6060833668571258e-05, "spike_contrast": 3.20486156963133, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 5.147274860064499e-05, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 1.4964238289394416e-05, 1.7345477317576297e-05, 1.5382447600131854e-05, 1.43398183354293e-05, 1.8841785276890732e-05, 1.7429230865673162e-05, 1.623117714188993e-05, 1.4482900951406918e-05, 1.2702618732873816e-05, 1.1208579053345602e-05, 1.0501379620109219e-05, 1.0047559953818563e-05, 1.4258240298659075e-05, 1.3366416169446893e-05, 1.2530163076007739e-05, 1.1682102922350168e-05, 1.4462313629337586e-05, 1.3515921637008432e-05, 1.3489259799825959e-05, 1.4416938029171433e-05, 6.775441579520702e-05, 5.7769335398916155e-05, 4.614627323462628e-05, 3.664553878479637e-05, 4.9048179789679125e-05, 4.286609328119084e-05, 3.200363789801486e-05, 2.0764351575053297e-05, 2.1946214474155568e-05, 1.55297511810204e-05, 1.0194041351496708e-05, 9.139850590145215e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 6.422088154067751e-05, "first_to_last_ratio": 0.637682021407296, "imbalance_abs_log_ratio": 0.44991551892756126, "last_quartile_mean": 0.00010070988264487823, "mean": 8.629204910448607e-05, "normalized": [ 0.6940783808537224, 0.8062729813904731, 0.7180303336815479, 0.6647219507645815, 0.8588589713696609, 0.794136864353616, 0.7473889560340541, 0.6703283692089702, 0.5917748510321142, 0.5226947352403994, 0.4893764215021349, 0.4670989784552037, 0.661952700545094, 0.6164859593679979, 0.5786990118719374, 0.539368710966703, 0.6707742418503443, 0.6266861813765591, 0.6317438172781369, 0.6712283354793382, 3.1389231589965445, 2.679577878793134, 2.1364533990925567, 1.686690164787602, 2.264487313238182, 1.9784244316759128, 1.477878596963198, 0.9605191983702851, 1.022047261980904, 0.7239260887312458, 0.4786954192739982, 0.4306763354738471 ], "peak_layer": 21, "peak_normalized": 3.1389231589965445, "population_cv": 0.690500620274875, "rest_mean": 6.42200389443638e-05, "spike_contrast": 3.1996384204494492, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0002054809039691463, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 5.98934457229916e-05, 6.957494770176709e-05, 6.196030881255865e-05, 5.736021921620704e-05, 7.411270053125918e-05, 6.852769729448482e-05, 6.449372449424118e-05, 5.784400855191052e-05, 5.106546450406313e-05, 4.51043997600209e-05, 4.2229294194839895e-05, 4.030692798551172e-05, 5.7121254940284416e-05, 5.319783667800948e-05, 4.993712354917079e-05, 4.654323129216209e-05, 5.788248381577432e-05, 5.40780347364489e-05, 5.451446850202046e-05, 5.7921668485505506e-05, 0.00027086411137133837, 0.00023122626589611173, 0.00018435894162394106, 0.00014554795052390546, 0.00019540725043043494, 0.0001707222982076928, 0.00012752917245961726, 8.288516983157024e-05, 8.819455251796171e-05, 6.24690655968152e-05, 4.13076086260844e-05, 3.716394348884933e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.056951718870550394, "first_to_last_ratio": 0.6376820191187264, "imbalance_abs_log_ratio": 0.4499155225164498, "last_quartile_mean": 0.08931052964180708, "mean": 0.07652465114369988, "normalized": [ 0.6940783685018588, 0.8062729890693022, 0.7180303052956253, 0.6647219248274491, 0.8588589447083164, 0.7941368847913186, 0.7473889396113633, 0.6703284088385635, 0.5917748342982165, 0.5226947125226558, 0.4893764209845987, 0.46709901042220126, 0.6619527107838585, 0.616485998695461, 0.5786990422522198, 0.5393687059717984, 0.6707742773702546, 0.6266862088911475, 0.6317438155472213, 0.671228324296238, 3.138923030946758, 2.6795778685275287, 2.136453429700619, 1.6866902298133055, 2.2644871659665227, 1.9784244587489672, 1.4778786878450325, 0.9605192092561265, 1.0220472167545944, 0.7239261000048334, 0.4786954380387232, 0.4306763367173204 ], "peak_layer": 21, "peak_normalized": 3.138923030946758, "population_cv": 0.6905006031140464, "rest_mean": 0.056950972587973984, "spike_contrast": 3.199638339154521, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.18222251534461975, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 0.05311410501599312, 0.0616997592151165, 0.05494701862335205, 0.05086761340498924, 0.06572388112545013, 0.060771048069000244, 0.05719367787241936, 0.051296647638082504, 0.045285362750291824, 0.03999903053045273, 0.03744935989379883, 0.03574458882212639, 0.05065570026636124, 0.04717637598514557, 0.04428474232554436, 0.04127500206232071, 0.05133076757192612, 0.04795694351196289, 0.048343975096940994, 0.05136551335453987, 0.24020498991012573, 0.20505376160144806, 0.163491353392601, 0.1290733814239502, 0.17328909039497375, 0.15139824151992798, 0.11309415102005005, 0.07350339740514755, 0.07821180671453476, 0.05539819225668907, 0.036632001399993896, 0.03295735642313957 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 2.591977136034984e-06, "first_to_last_ratio": 0.377584641115206, "imbalance_abs_log_ratio": 0.9739605205507381, "last_quartile_mean": 6.864625447633443e-06, "mean": 4.875532418679995e-06, "normalized": [ 0.5891470886653372, 0.5445974877366685, 0.5278757119769764, 0.5390247128297342, 0.5219320902964267, 0.5098223481982903, 0.5047854170324074, 0.5158515929518226, 0.43682532071303765, 0.4047462819890774, 0.3956760190748421, 0.40124851416781243, 0.5110306315304836, 0.5086020328656453, 0.5111981468235955, 0.5112038363741968, 0.5662160078662817, 0.5872378713533353, 0.609816433193417, 0.6494206487580901, 3.0257058078799917, 2.8686033250815077, 2.442085511360172, 2.0535509509100836, 2.6979608311079244, 2.429139638501957, 1.8298842703872822, 1.2190711864897739, 1.1821681083908706, 0.8593060718331675, 0.5582216762792698, 0.48804442738052095 ], "peak_layer": 21, "peak_normalized": 3.0257058078799917, "population_cv": 0.8235699242655958, "rest_mean": 3.415056493462611e-06, "spike_contrast": 3.7370106290435205, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.2762102414853871e-05, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 2.872405730158789e-06, 2.6552027065918082e-06, 2.573675146777532e-06, 2.6280324618710438e-06, 2.544696826589643e-06, 2.4856553864083253e-06, 2.4610976652184036e-06, 2.515051164664328e-06, 2.129756012436701e-06, 1.973353619177942e-06, 1.929131258293637e-06, 1.956300138772349e-06, 2.491546410965384e-06, 2.4797056994430022e-06, 2.492363137207576e-06, 2.49239087679598e-06, 2.7606045023276238e-06, 2.8630972792598186e-06, 2.9731797894783085e-06, 3.1662714263802627e-06, 1.4751926755707245e-05, 1.398596850776812e-05, 1.1906467079825234e-05, 1.0012154234573245e-05, 1.315399549639551e-05, 1.1843349057016894e-05, 8.921660082705785e-06, 5.943621090409579e-06, 5.7636989367892966e-06, 4.189574610791169e-06, 2.7216278795094695e-06, 2.3794764274498448e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 4.862782361669815e-05, "first_to_last_ratio": 0.7782948608369358, "imbalance_abs_log_ratio": 0.25064982810047876, "last_quartile_mean": 6.247994951991132e-05, "mean": 6.004880788168521e-05, "normalized": [ 0.7686076787185943, 0.8690658694356951, 0.7879558094938062, 0.7385895471539009, 0.9274752179545845, 0.8617380288980451, 0.805485823509466, 0.7195218583140428, 0.6496432282856196, 0.579963373368429, 0.542973633406911, 0.5176967820022496, 0.7209744736270646, 0.6651136389238139, 0.6225458367735092, 0.578423456947426, 0.7443181598307745, 0.6978871776308329, 0.7018192413444355, 0.7379772277240845, 3.17488720924795, 2.647274451154299, 2.058197351015695, 1.557976178868823, 2.137956515672529, 1.7650930593990968, 1.3105561147977765, 0.8840594863751997, 0.8342799220515003, 0.5872955759322932, 0.4206210094039151, 0.3840270627376372 ], "peak_layer": 21, "peak_normalized": 3.17488720924795, "population_cv": 0.6535433957622422, "rest_mean": 4.5422938354830774e-05, "spike_contrast": 3.060755388404179, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00013902850332669914, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 4.615397483576089e-05, 5.218636943027377e-05, 4.7315807023551315e-05, 4.4351421820465475e-05, 5.569378117797896e-05, 5.17463413416408e-05, 4.8368463467340916e-05, 4.320642983657308e-05, 3.9010301406960934e-05, 3.482610918581486e-05, 3.260491939727217e-05, 3.108707460341975e-05, 4.329365765443072e-05, 3.9939281123224646e-05, 3.738313534995541e-05, 3.47336390404962e-05, 4.4695418182527646e-05, 4.19072930526454e-05, 4.2143408791162074e-05, 4.43146527686622e-05, 0.00019064819207414985, 0.00015896567492745817, 0.0001235922973137349, 9.355461224913597e-05, 0.00012838174006901681, 0.00010599173401715234, 7.869733235565946e-05, 5.308671825332567e-05, 5.009751475881785e-05, 3.526639920892194e-05, 2.5257790184696205e-05, 2.3060367311700247e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 3.645344395408756e-05, "first_to_last_ratio": 0.9865245275167812, "imbalance_abs_log_ratio": 0.01356709065898008, "last_quartile_mean": 3.695138127568498e-05, "mean": 4.1334421752026174e-05, "normalized": [ 0.8559009893021382, 0.9469134291395908, 0.8691128415923758, 0.8158459610499553, 1.0100395593992384, 0.9426509450849102, 0.8490005706596633, 0.7658551995239993, 0.720151565776352, 0.6499584076998435, 0.6111014164691825, 0.5791973878203037, 0.798669989787535, 0.7292203680582068, 0.6791147051708346, 0.6276623067938342, 0.8453470506184937, 0.7894265691211152, 0.7857081832788244, 0.8180697050541997, 3.276793902438668, 2.6146895715088023, 1.9277214837041228, 1.3401559674726915, 1.9403857172500238, 1.4668533992566692, 1.0945373428822949, 0.8102509556533821, 0.6598620161196013, 0.46946557551492035, 0.3666267887349235, 0.3437101280633033 ], "peak_layer": 21, "peak_normalized": 3.276793902438668, "population_cv": 0.6207652283001258, "rest_mean": 3.1996292111418574e-05, "spike_contrast": 2.867842357851226, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.176032181130722e-05, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 3.5378172469791025e-05, 3.9140119042713195e-05, 3.5924276744481176e-05, 3.372252103872597e-05, 4.174940113443881e-05, 3.8963931729085743e-05, 3.509294765535742e-05, 3.165618181810714e-05, 2.976704854518175e-05, 2.6865654945140705e-05, 2.5259523681597784e-05, 2.3940789105836302e-05, 3.301256219856441e-05, 3.0141902243485674e-05, 2.8070813641534187e-05, 2.5944058506865986e-05, 3.4941931517096236e-05, 3.263049075030722e-05, 3.2476793421665207e-05, 3.381443821126595e-05, 0.0001354443811578676, 0.00010807668149936944, 7.968125282786787e-05, 5.53945719730109e-05, 8.020472159842029e-05, 6.06315370532684e-05, 4.524206815403886e-05, 3.349125472595915e-05, 2.7275014872429892e-05, 1.940508809639141e-05, 1.5154306311160326e-05, 1.4207059393811505e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00012982848875253694, "first_to_last_ratio": 0.6036924070795879, "imbalance_abs_log_ratio": 0.5046904705661434, "last_quartile_mean": 0.00021505734912352636, "mean": 0.00017907720302900998, "normalized": [ 0.6739415088564852, 0.7635979187076084, 0.6884132846665824, 0.6448880523181134, 0.847636297931654, 0.7820437425942102, 0.7390348805248977, 0.6603333703627584, 0.580097091853396, 0.5130765445550399, 0.4840585582545864, 0.4614791311662735, 0.637447225260648, 0.5884103323685723, 0.5619029319457929, 0.5245290839622635, 0.6356966288449003, 0.6100272686595811, 0.6118819192326126, 0.6643605628507145, 3.097964891544288, 2.6928767346712488, 2.1827809090098653, 1.7461631642015993, 2.367937988760414, 2.0449314566347367, 1.563314534665903, 1.0260809297257236, 1.0359670091642164, 0.6947146354097079, 0.460142110411202, 0.41426930088440395 ], "peak_layer": 21, "peak_normalized": 3.097964891544288, "population_cv": 0.7095182139952283, "rest_mean": 0.00013206795362445214, "spike_contrast": 3.278063587209748, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0004329271498136222, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 0.00012068756041117013, 0.00013674297952093184, 0.0001232791255461052, 0.0001154847486759536, 0.00015179233741946518, 0.0001400462060701102, 0.00013234429934527725, 0.00011825065303128213, 0.00010388216469436884, 9.188031253870577e-05, 8.668385271448642e-05, 8.264039206551388e-05, 0.00011415226617828012, 0.00010537087655393407, 0.00010062400542665273, 9.393120126333088e-05, 0.00011383877426851541, 0.00010924197704298422, 0.00010957410268019885, 0.00011897183139808476, 0.0005547748878598213, 0.0004822328337468207, 0.00039088630001060665, 0.0003126980154775083, 0.0004240437119733542, 0.00036620060564018786, 0.0002799539943225682, 0.00018374770297668874, 0.00018551807443145663, 0.0001244075538124889, 8.240096212830395e-05, 7.418618770316243e-05 ] } } }, "post_mlp_state": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "89f2226456bfe96e245d2aad0f699d32c676f9f53099789cd870253d002402f5", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.16915453970432281, 0.374446302652359, 0.39966297149658203, 0.4194087088108063, 0.09892481565475464, 0.11664740741252899, 0.13951337337493896, 0.18593810498714447, 0.06607726216316223, 0.10425819456577301, 0.14551182091236115, 0.19865906238555908, 0.06484340131282806, 0.10430748760700226, 0.14720691740512848, 0.20439592003822327, 0.10052042454481125, 0.14085541665554047, 0.1745898574590683, 0.2420409768819809, 0.049456026405096054, 0.08509506285190582, 0.11783231794834137, 0.15649692714214325, 0.05533855780959129, 0.08507220447063446, 0.11942370235919952, 0.1559232622385025, 0.06628894805908203, 0.12384890764951706, 0.19928614795207977, 0.30097496509552 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.5986864127626177e-05, "first_to_last_ratio": 0.8170274391577745, "imbalance_abs_log_ratio": 0.20208259942567244, "last_quartile_mean": 1.956710798367567e-05, "mean": 1.9039307829871177e-05, "normalized": [ 0.7853167979867789, 0.9097665173048055, 0.808777694116954, 0.752402497635975, 0.9858540561935626, 0.9044938358034963, 0.8288447236919866, 0.7419578801196516, 0.6090850929460484, 0.5628647453138417, 0.5328979550345672, 0.5131403750850531, 0.7265824660927316, 0.6757007408890954, 0.6225826951824261, 0.5898791530514104, 0.7532772493405897, 0.6927950381940383, 0.6897848547806317, 0.8094505726944553, 3.2076147584918284, 2.5002121502474965, 1.982484105644761, 1.5924612114390058, 2.273889158865966, 1.6940625327896233, 1.0968336406315982, 0.8719471681044353, 0.815653609383569, 0.5388945327422838, 0.4819664542042268, 0.448525735997107 ], "peak_layer": 21, "peak_normalized": 3.2076147584918284, "population_cv": 0.6428608089466793, "rest_mean": 1.4415815443229534e-05, "spike_contrast": 3.052631111367605, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.400616671773605e-05, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 1.4951888260839041e-05, 1.7321324776276015e-05, 1.5398567484226078e-05, 1.4325222764455248e-05, 1.8769978851196356e-05, 1.7220936570083722e-05, 1.5780629837536253e-05, 1.4126364476396702e-05, 1.1596558579185512e-05, 1.0716555152612273e-05, 1.0146008207811974e-05, 9.769837561179884e-06, 1.3833627235726453e-05, 1.286487440665951e-05, 1.1853543583129067e-05, 1.1230890777369495e-05, 1.4341877431434114e-05, 1.3190337995183654e-05, 1.3133026186551433e-05, 1.541137862659525e-05, 6.107076478656381e-05, 4.760230876854621e-05, 3.7745125155197456e-05, 3.03193592117168e-05, 4.329327566665597e-05, 3.225377804483287e-05, 2.0882953322143294e-05, 1.6601270544924773e-05, 1.5529480151599273e-05, 1.0260178896714933e-05, 9.176307685265783e-06, 8.539619557268452e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 6.365272747643758e-05, "first_to_last_ratio": 0.8123633197064085, "imbalance_abs_log_ratio": 0.2078076008289402, "last_quartile_mean": 7.835499945940683e-05, "mean": 7.614457786075945e-05, "normalized": [ 0.7854618236676242, 0.9130540849409994, 0.8147131127464502, 0.7522410092502885, 0.9690072688462938, 0.8892440039434748, 0.8242354060534288, 0.7396083689563772, 0.6126026690255093, 0.5670644008562938, 0.5359283946772616, 0.5145269561429746, 0.7290174418180176, 0.6731434248219464, 0.6214223500409742, 0.5883307730946398, 0.7535191463387267, 0.6934458909688136, 0.6978155985835179, 0.8168473226649812, 3.219066994145898, 2.5077095541323673, 1.973834673592298, 1.5759251613516239, 2.2647260999477155, 1.6880724220748888, 1.092454232265147, 0.8732751258075867, 0.8206258345435784, 0.5459870832104624, 0.48994656248557816, 0.45714680900426247 ], "peak_layer": 21, "peak_normalized": 3.219066994145898, "population_cv": 0.6420341358971389, "rest_mean": 5.7697108584559625e-05, "spike_contrast": 3.04626897714722, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00017576091195223854, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 5.980865898891352e-05, 6.95241178618744e-05, 6.203598604770377e-05, 5.727907409891486e-05, 7.378464943030849e-05, 6.77111092954874e-05, 6.276105705183e-05, 5.631716703646816e-05, 4.664637162932195e-05, 4.3178879423066974e-05, 4.080804137629457e-05, 3.917843787348829e-05, 5.551072536036372e-05, 5.1256221922812983e-05, 4.731794251711108e-05, 4.47981983597856e-05, 5.737639730796218e-05, 5.2802144637098536e-05, 5.3134874178795144e-05, 6.219849456101656e-05, 0.0002451144973747432, 0.00019094848539680243, 0.00015029680798761547, 0.00011999815615126863, 0.0001724466128507629, 0.00012853756197728217, 8.318446634802967e-05, 6.64951658109203e-05, 6.248620775295421e-05, 4.157395596848801e-05, 3.7306774174794555e-05, 3.4809250792022794e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.05644787335768342, "first_to_last_ratio": 0.8123632868395458, "imbalance_abs_log_ratio": 0.2078076412872703, "last_quartile_mean": 0.06948599754832685, "mean": 0.06752577290171757, "normalized": [ 0.7854618348329956, 0.9130540359301327, 0.8147131249195937, 0.7522410113175043, 0.9690071730110875, 0.88924395208314, 0.8242353599678512, 0.7396083806676593, 0.6126026750590019, 0.5670643980854334, 0.5359283892166958, 0.5145269586503932, 0.7290174229589911, 0.6731434525628229, 0.6214223402600404, 0.5883307763560162, 0.7535191533057549, 0.6934459281737582, 0.6978155981634587, 0.8168473704719704, 3.2190670201887848, 2.507709505453073, 1.9738347607250095, 1.5759251284181852, 2.264726151347608, 1.68807248858185, 1.0924542395938426, 0.8732750663142743, 0.820625855657368, 0.5459870819951741, 0.48994655813463567, 0.45714680759589343 ], "peak_layer": 21, "peak_normalized": 3.2190670201887848, "population_cv": 0.6420341436220799, "rest_mean": 0.05116637251167386, "spike_contrast": 3.0462690113979045, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.15586653500795364, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 0.05303891748189926, 0.06165467947721481, 0.05501413345336914, 0.05079565569758415, 0.06543295830488205, 0.060046885162591934, 0.05565712973475456, 0.04994262754917145, 0.041366469115018845, 0.038291461765766144, 0.03618897870182991, 0.03474383056163788, 0.04922746494412422, 0.04545453190803528, 0.041962023824453354, 0.03972749039530754, 0.05088196322321892, 0.04682547226548195, 0.04712053760886192, 0.05515825003385544, 0.21736998856067657, 0.1693350225687027, 0.13328471779823303, 0.10641556233167648, 0.15292738378047943, 0.11398839950561523, 0.07376881688833237, 0.058968573808670044, 0.055413395166397095, 0.036868199706077576, 0.033084020018577576, 0.030869191512465477 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 2.579253447265728e-06, "first_to_last_ratio": 0.47590496457778414, "imbalance_abs_log_ratio": 0.7425370989326324, "last_quartile_mean": 5.419681741614113e-06, "mean": 4.2137082871818166e-06, "normalized": [ 0.6771144196047112, 0.6338426021784607, 0.6121020910961615, 0.6258737201152189, 0.6050362914422536, 0.5852204423685657, 0.5784631886764509, 0.5792281322851218, 0.4697078599262646, 0.4522117174548025, 0.44580914592548354, 0.4558904724294877, 0.5773213311917524, 0.5667240346830847, 0.5644321177223985, 0.5681434108598962, 0.6503548834184826, 0.6631947770191623, 0.6837887360559592, 0.7582966501774209, 3.020973835679826, 2.627473806530212, 2.3058644468737186, 2.00331262981993, 2.819406877957528, 2.1402761714678284, 1.4825745701937771, 1.1227288221975413, 0.9855094000056028, 0.6429466506282437, 0.5709200007532207, 0.5252567632614316 ], "peak_layer": 21, "peak_normalized": 3.020973835679826, "population_cv": 0.7577358346351384, "rest_mean": 2.999999306094078e-06, "spike_contrast": 3.589246425218318, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.0767736785055605e-05, "top_five_layers": [ 21, 25, 22, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 2.8531626412586775e-06, 2.670827825568267e-06, 2.5792196538532153e-06, 2.6372492811788106e-06, 2.5494464352959767e-06, 2.465948227836634e-06, 2.43747513195558e-06, 2.440698381178663e-06, 1.979211901925737e-06, 1.9054882614000235e-06, 1.8785096926876577e-06, 1.920989461723366e-06, 2.4326636776095256e-06, 2.3880097614892293e-06, 2.3783522919984534e-06, 2.393990598648088e-06, 2.7404057618696243e-06, 2.7945093279413413e-06, 2.8812862638005754e-06, 3.1952408789948095e-06, 1.2729502486763522e-05, 1.1071408152929507e-05, 9.716240128909703e-06, 8.441375030088238e-06, 1.1880158126587048e-05, 9.018499440571759e-06, 6.247136752790539e-06, 4.7308517423516605e-06, 4.152649125899188e-06, 2.709189629968023e-06, 2.405690338491695e-06, 2.2132787762529915e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 4.826905797017389e-05, "first_to_last_ratio": 0.9723593605402814, "imbalance_abs_log_ratio": 0.028029830358518852, "last_quartile_mean": 4.964117169947713e-05, "mean": 5.395593586854375e-05, "normalized": [ 0.8546740011831107, 0.9667023865336807, 0.8780594173977533, 0.8215201862718362, 1.028762848378348, 0.9492707323927138, 0.8750383055629682, 0.7827836490446655, 0.6672270484578459, 0.6202534690629704, 0.5854228619045188, 0.5620710233383177, 0.781592991024926, 0.7155185842665527, 0.6597031604820425, 0.621567177701999, 0.8222052916650392, 0.7616961441483728, 0.7641199379358804, 0.8753280307745216, 3.236716477578425, 2.479496544122304, 1.8830004075567146, 1.4470156791415152, 2.05665793704697, 1.5065644924277526, 1.0009572516438536, 0.8155596816984735, 0.6595753226879756, 0.4759799891964506, 0.4318790517932509, 0.413079917578251 ], "peak_layer": 21, "peak_normalized": 3.236716477578425, "population_cv": 0.6107674157639074, "rest_mean": 4.176012171902349e-05, "spike_contrast": 2.869084843241085, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00011981333227595314, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 4.61147355963476e-05, 5.2159331971779466e-05, 4.737651761388406e-05, 4.4325890485197306e-05, 5.550786227104254e-05, 5.121879075886682e-05, 4.72135106974747e-05, 4.2235824366798624e-05, 3.6000859836349264e-05, 3.3466356399003416e-05, 3.158703839289956e-05, 3.0327068088809028e-05, 4.217158129904419e-05, 3.860647484543733e-05, 3.559490141924471e-05, 3.353723877808079e-05, 4.4362855987856165e-05, 4.1098028304986656e-05, 4.122880636714399e-05, 4.7229143092408776e-05, 0.00017464006668888032, 0.00013378355652093887, 0.00010159904923057184, 7.807508518453687e-05, 0.00011096890375483781, 8.128809713525698e-05, 5.400758527684957e-05, 4.4004285882692784e-05, 3.558800381142646e-05, 2.5681945771793835e-05, 2.330243842152413e-05, 2.228811354143545e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 3.626825309765991e-05, "first_to_last_ratio": 1.1971952602722011, "imbalance_abs_log_ratio": 0.17998153797771219, "last_quartile_mean": 3.0294350722215313e-05, "mean": 3.794959516767449e-05, "normalized": [ 0.9342317076920968, 1.032527492584725, 0.9483990781630639, 0.8885231558783968, 1.1008949985426397, 1.0148066560537, 0.9027782413985963, 0.8234017777861798, 0.7333936750261243, 0.6823073758552121, 0.6464481216407498, 0.616777331910669, 0.8501710687237772, 0.7702178906455867, 0.7042577074377577, 0.660638930723233, 0.914060026408416, 0.847914539167618, 0.8394293783449699, 0.9442746494437388, 3.300220154812763, 2.4489522165880904, 1.7640184700183157, 1.245126339879113, 1.7643693302152461, 1.271277790934411, 0.9095623054467832, 0.780451313056054, 0.5159478762516835, 0.40747033934370974, 0.37633772798986104, 0.36081233203671814 ], "peak_layer": 21, "peak_normalized": 3.300220154812763, "population_cv": 0.5858778602410362, "rest_mean": 3.018723526582579e-05, "spike_contrast": 2.6456990159702416, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 7.986633863765746e-05, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 3.5453715099720284e-05, 3.9184000343084335e-05, 3.599136107368395e-05, 3.371909406268969e-05, 4.1778519516810775e-05, 3.85115017706994e-05, 3.4260068787261844e-05, 3.124776412732899e-05, 2.783199306577444e-05, 2.5893288693623617e-05, 2.453244451317005e-05, 2.3406450054608285e-05, 3.226364788133651e-05, 2.9229457140900195e-05, 2.6726294890977442e-05, 2.5070979972952046e-05, 3.468820796115324e-05, 3.217801349819638e-05, 3.185600508004427e-05, 3.5834840673487633e-05, 0.00012524201883934438, 9.293674520449713e-05, 6.694378680549562e-05, 4.725204053102061e-05, 6.695710180792958e-05, 4.8244477511616424e-05, 3.451752127148211e-05, 2.9617811378557235e-05, 1.95800130313728e-05, 1.546333442092873e-05, 1.4281864423537627e-05, 1.3692681932298001e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 0.00012846494701079791, "first_to_last_ratio": 0.7741545804954827, "imbalance_abs_log_ratio": 0.25598370893064326, "last_quartile_mean": 0.00016594224234722788, "mean": 0.000157460202672155, "normalized": [ 0.7615886502877881, 0.869859991005712, 0.7823534284439785, 0.735201115807512, 0.9596786286759813, 0.8813992955944876, 0.8128285893567467, 0.7239431731448671, 0.5977864740547744, 0.5606030792342883, 0.5318673348078621, 0.5130567490014816, 0.6987134321316841, 0.6407269063222457, 0.6075335461477536, 0.5704889146486948, 0.7180428901779269, 0.6784140170824181, 0.6819219612414157, 0.7908928021790416, 3.2176296651190497, 2.545498211708922, 2.0288816729118655, 1.6601468184271124, 2.321862751456626, 1.7738386620365922, 1.1711907122839438, 0.9284857615000071, 0.7896970255450443, 0.5236030848875767, 0.47341239614817127, 0.4488522586284289 ], "peak_layer": 21, "peak_normalized": 3.2176296651190497, "population_cv": 0.6625973254291921, "rest_mean": 0.0001179550758765631, "spike_contrast": 3.143466990402101, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00037078788736835124, "top_five_layers": [ 21, 22, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 0.00011991990322712809, 0.00013696833048015833, 0.00012318952940404415, 0.00011576491669984534, 0.0001511111913714558, 0.00013878531171940267, 0.00012798815441783518, 0.00011399223876651376, 9.412757935933769e-05, 8.82726744748652e-05, 8.374793833354488e-05, 8.078601968009025e-05, 0.000110019558633212, 0.00010088898852700368, 9.566235530655831e-05, 8.982930012280121e-05, 0.0001130631790147163, 0.00010682320862542838, 0.00010737557022366673, 0.00012453414092306048, 0.0005066486191935837, 0.00040081466431729496, 0.00031946811941452324, 0.0002614070544950664, 0.0003656009794212878, 0.00027930899523198605, 0.00018441592692397535, 0.00014619955618400127, 0.00012434585369192064, 8.24466478661634e-05, 7.454361184500158e-05, 7.067636761348695e-05 ] } } }, "pre_attention_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "ee695e78937de01b7128d3866b8fc5dd328ebdf2c159be37918d4f02952d5248", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.0626930519938469, 0.09030644595623016, 0.20229491591453552, 0.20300470292568207, 0.19049881398677826, 0.12695962190628052, 0.12132706493139267, 0.11585728824138641, 0.13427142798900604, 0.10485932976007462, 0.10419609397649765, 0.10852954536676407, 0.1286265105009079, 0.09096142649650574, 0.10080574452877045, 0.10566592961549759, 0.1287592649459839, 0.09650235623121262, 0.1065765842795372, 0.11446042358875275, 0.12395420670509338, 0.07221997529268265, 0.07677769660949707, 0.08416415005922318, 0.09094838052988052, 0.059439923614263535, 0.062182098627090454, 0.06896103918552399, 0.0776243805885315, 0.06285123527050018, 0.07231374084949493, 0.08016443997621536 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 1.0311521975836513e-05, "first_to_last_ratio": 1.1418129560476864, "imbalance_abs_log_ratio": 0.13261731152138012, "last_quartile_mean": 9.030832870848826e-06, "mean": 1.3945096057454975e-05, "normalized": [ 0.7145337044030123, 0.932335477800909, 0.645487324949067, 0.558162738870377, 0.34661223918607953, 0.7511338800577496, 0.9447212820009926, 1.0225104899919193, 0.6315142693979814, 0.5614582240786927, 0.4574638139025647, 0.40615350248375953, 0.4194348381092698, 0.7040945119904685, 0.6483274463519548, 0.5709286433652233, 0.8217341836062412, 1.1893729541204203, 1.2408864574521172, 1.5700693567578727, 2.5502861886942485, 4.114342551839228, 2.785136777749787, 2.232505646034574, 1.1806687356339707, 1.2053474712098855, 0.7898562388784587, 0.5945521897391881, 0.46830824879649285, 0.2867747481020556, 0.32588927285831853, 0.32939659158711965 ], "peak_layer": 22, "peak_normalized": 4.114342551839228, "population_cv": 0.8262244130468728, "rest_mean": 9.884005235276331e-06, "spike_contrast": 3.629600110811422, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 3.587498649721965e-05, "top_five_layers": [ 22, 23, 21, 24, 20 ], "top_five_spike_overlap": 4 }, "values": [ 9.964241144189145e-06, 1.3001507795706857e-05, 9.001382750284392e-06, 7.783633009239566e-06, 4.833540970139438e-06, 1.0474634109414183e-05, 1.3174229025025852e-05, 1.425900700269267e-05, 8.80652714840835e-06, 7.82958886702545e-06, 6.3793768276809715e-06, 5.663849606207805e-06, 5.849059107276844e-06, 9.818665603233967e-06, 9.040988516062498e-06, 7.961654773680493e-06, 1.1459162124083377e-05, 1.658592009334825e-05, 1.730428084556479e-05, 2.189476799685508e-05, 3.556398587534204e-05, 5.7374902098672464e-05, 3.883899989887141e-05, 3.113250568276271e-05, 1.6464538930449635e-05, 1.68086862686323e-05, 1.1014621122740209e-05, 8.291087397083174e-06, 6.530603513965616e-06, 3.999101409135619e-06, 4.544557214103406e-06, 4.593467110680649e-06 ] }, "element_rms": { "statistics": { "first_quartile_mean": 3.944479362871789e-05, "first_to_last_ratio": 1.1135282861542686, "imbalance_abs_log_ratio": 0.1075336103151947, "last_quartile_mean": 3.54232524841791e-05, "mean": 5.533300526394669e-05, "normalized": [ 0.5280207224829799, 0.8999098868607942, 0.6284748654319019, 0.5778568458898549, 0.34988006202074273, 0.7764713506345448, 0.9190909808587969, 1.023190827540606, 0.6226417920568781, 0.5519145939388094, 0.4568960180345979, 0.4213932325876586, 0.4074494455716704, 0.7465319443901276, 0.6474109861274324, 0.5764712279753077, 0.7979703549327781, 1.2369697451324837, 1.2324489817728508, 1.5808985864286207, 2.5860289117498976, 4.232714993086349, 2.835646318943954, 2.2422528664695807, 1.1646676435019565, 1.2175941057769888, 0.783401544111113, 0.5289102464485725, 0.46735876466607296, 0.2966943922188728, 0.3266155865457417, 0.3362221758114643 ], "peak_layer": 22, "peak_normalized": 4.232714993086349, "population_cv": 0.8496980089222349, "rest_mean": 3.881239232820614e-05, "spike_contrast": 3.7241794809927478, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00014454431511694564, "top_five_layers": [ 22, 23, 21, 24, 20 ], "top_five_spike_overlap": 4 }, "values": [ 2.921697341662366e-05, 4.9794718506745994e-05, 3.477540303720161e-05, 3.197455589543097e-05, 1.9359915313543752e-05, 4.296449333196506e-05, 5.085606608190574e-05, 5.661622344632633e-05, 3.4452641557436436e-05, 3.053909313166514e-05, 2.5281429770984687e-05, 2.3316953956964426e-05, 2.2545402316609398e-05, 4.1307856008643284e-05, 3.582319550332613e-05, 3.189788549207151e-05, 4.415409784996882e-05, 6.844525341875851e-05, 6.819510599598289e-05, 8.747586980462074e-05, 0.0001430927513865754, 0.00023420884099323303, 0.00015690483269281685, 0.00012407058966346085, 6.444456084864214e-05, 6.737314106430858e-05, 4.3347961764084175e-05, 2.9266193450894207e-05, 2.5860364985419437e-05, 1.641699236643035e-05, 1.807262196962256e-05, 1.8604183424031362e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.03498003748245537, "first_to_last_ratio": 1.1135283344223057, "imbalance_abs_log_ratio": 0.10753365366212776, "last_quartile_mean": 0.03141369321383536, "mean": 0.04906986153218895, "normalized": [ 0.528020735774383, 0.8999099163028887, 0.6284749107820657, 0.5778568640801447, 0.34988006993508275, 0.776471348923419, 0.919091002023713, 1.0231908018037408, 0.622641782085556, 0.5519145889381298, 0.45689603549337854, 0.4213932430966296, 0.4074494429214576, 0.7465319861987265, 0.6474110467766695, 0.5764712070514831, 0.7979703658517325, 1.236969773835328, 1.2324490791670337, 1.5808986244806464, 2.5860289675304746, 4.232714870782151, 2.835646138092529, 2.242252864087746, 1.1646675839359508, 1.2175940341247224, 0.7834015320190452, 0.528910268066999, 0.467358763362216, 0.2966943974186558, 0.32661557776013733, 0.33622217729716364 ], "peak_layer": 22, "peak_normalized": 4.232714870782151, "population_cv": 0.8496979833458974, "rest_mean": 0.034419217595347655, "spike_contrast": 3.72417933196884, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.12818333879113197, "top_five_layers": [ 22, 23, 21, 24, 20 ], "top_five_spike_overlap": 4 }, "values": [ 0.0259099043905735, 0.0441584549844265, 0.03083917684853077, 0.02835535630583763, 0.017168566584587097, 0.03810134157538414, 0.04509966820478439, 0.05020783096551895, 0.030552946031093597, 0.027082372456789017, 0.022419825196266174, 0.020677708089351654, 0.019993487745523453, 0.036632221192121506, 0.03176837041974068, 0.028287362307310104, 0.039156295359134674, 0.06069793552160263, 0.06047610566020012, 0.0775744765996933, 0.12689608335494995, 0.2076987326145172, 0.13914476335048676, 0.11002703756093979, 0.05715007707476616, 0.05974717065691948, 0.038441404700279236, 0.02595355361700058, 0.022933229804039, 0.014558752998709679, 0.01602698117494583, 0.016498375684022903 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 2.750052800593039e-06, "first_to_last_ratio": 0.4308304317433478, "imbalance_abs_log_ratio": 0.8420406961220296, "last_quartile_mean": 6.383144267374519e-06, "mean": 4.5530042669383874e-06, "normalized": [ 2.0777537806616246, 0.8353410859824805, 0.1910970826520623, 0.2941663520688202, 0.18051642612757607, 0.3343816911052641, 0.3564217354179324, 0.562389007550559, 0.34578362377189115, 0.35956396375636174, 0.3051237753595574, 0.33780495264365074, 0.3038063276744268, 0.5221079484446093, 0.49376666786286805, 0.5929821493588614, 0.3686987288124583, 0.7392110518834059, 0.5965749796632107, 0.9039444986896679, 1.6537371809131796, 3.0803086468030596, 2.8630898340929525, 2.485723197420275, 1.724168620325731, 2.41211172564755, 1.9299601312238395, 1.6334426636441552, 1.3339260437132254, 0.9558102182238872, 0.6013349904117006, 0.6249509180931564 ], "peak_layer": 22, "peak_normalized": 3.0803086468030596, "population_cv": 0.8333981127885269, "rest_mean": 3.4051366684353625e-06, "spike_contrast": 3.1574325337710922, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.0751489298854722e-05, "top_five_layers": [ 22, 23, 24, 26, 1 ], "top_five_spike_overlap": 3 }, "values": [ 9.460021828999743e-06, 3.80331152882718e-06, 8.700658327143174e-07, 1.3393406561590382e-06, 8.218920584113221e-07, 1.5224412663883413e-06, 1.622789682187431e-06, 2.56055955105694e-06, 1.5743543144708383e-06, 1.6370962612199946e-06, 1.389229851156415e-06, 1.5380273907794617e-06, 1.383231506224547e-06, 2.3771597170707537e-06, 2.248121745651588e-06, 2.699850256249192e-06, 1.678686885497882e-06, 3.3656310733931605e-06, 2.71620842795528e-06, 4.1156631596095394e-06, 7.529472441092366e-06, 1.4024658412381541e-05, 1.3035660231253132e-05, 1.1317508324282244e-05, 7.850147085264325e-06, 1.0982354979205411e-05, 8.787116712483112e-06, 7.437071417371044e-06, 6.073370968806557e-06, 4.35180800195667e-06, 2.737880777203827e-06, 2.8454041967052035e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 2.6044932951663213e-05, "first_to_last_ratio": 1.1246393261284713, "imbalance_abs_log_ratio": 0.11746238525662386, "last_quartile_mean": 2.315847609679622e-05, "mean": 3.690820258839267e-05, "normalized": [ 0.45689793146127605, 0.8738188272728166, 0.6345310201461108, 0.5870234156151962, 0.3959041381169584, 0.8145991894129532, 0.9605505534893847, 0.9220179377165436, 0.6534489876582346, 0.574567881594872, 0.47330901138441, 0.4295574998000546, 0.4505301728221963, 0.8093943395842734, 0.6650197233282829, 0.5723463492360998, 0.7892367771238881, 1.2091363576952083, 1.2417470885367035, 1.5367390457984618, 2.460873737770247, 4.227774989574034, 3.004456821926061, 2.2368262162922274, 1.1669129504749511, 1.3012136399895533, 0.7651380597325796, 0.47754123850698527, 0.4891606669175266, 0.2974921422063077, 0.24654864385331124, 0.27568464496229084 ], "peak_layer": 22, "peak_normalized": 4.227774989574034, "population_cv": 0.8549059646786937, "rest_mean": 2.5840054991864168e-05, "spike_contrast": 3.7413310319999478, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 9.667619960964657e-05, "top_five_layers": [ 22, 23, 21, 24, 20 ], "top_five_spike_overlap": 4 }, "values": [ 1.6863281416590326e-05, 3.2251082302536815e-05, 2.341939944017213e-05, 2.166597914765589e-05, 1.4612110135203693e-05, 3.006539191119373e-05, 3.545219442457892e-05, 3.4030024835374206e-05, 2.4117627617670223e-05, 2.120626777468715e-05, 1.7468984879087657e-05, 1.5854195225983858e-05, 1.6628258890705183e-05, 2.9873290259274654e-05, 2.4544682673877105e-05, 2.1124275008332916e-05, 2.9129310860298574e-05, 4.462704964680597e-05, 4.583065310725942e-05, 5.671827602782287e-05, 9.082642645807937e-05, 0.00015603957581333816, 0.00011088910105172545, 8.255723514594138e-05, 4.3068659579148516e-05, 4.8025456635514274e-05, 2.823987051669974e-05, 1.7625188775127754e-05, 1.805404099286534e-05, 1.0979900253005326e-05, 9.099667295231484e-06, 1.0175024726777337e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 1.700510824775847e-05, "first_to_last_ratio": 1.1234226056354075, "imbalance_abs_log_ratio": 0.1163799234305708, "last_quartile_mean": 1.5136875617827172e-05, "mean": 2.416565462226572e-05, "normalized": [ 0.4144047676027832, 0.875113510868426, 0.648392503192907, 0.5978987553197436, 0.4472461776170949, 0.8581546509904864, 0.9915956817879598, 0.7967069167230741, 0.6805807808463835, 0.6196150111411964, 0.5091007553108428, 0.44255818216184833, 0.49857092482915655, 0.8652430602969526, 0.6897944860164179, 0.5800177420993363, 0.7901724317041942, 1.1871172558602836, 1.243251110659502, 1.5081298165112624, 2.2807123105787017, 4.096637658310102, 3.1477300512572848, 2.2202178168572866, 1.1479124533540228, 1.3987972507734237, 0.7581679674914413, 0.4356681137302007, 0.511763265149204, 0.3118761594955322, 0.21315449183792956, 0.2336979396250189 ], "peak_layer": 22, "peak_normalized": 4.096637658310102, "population_cv": 0.8395518179977153, "rest_mean": 1.710104003938829e-05, "spike_contrast": 3.6439054716132255, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 6.231457336980384e-05, "top_five_layers": [ 22, 23, 21, 24, 20 ], "top_five_spike_overlap": 4 }, "values": [ 1.001436248770915e-05, 2.114769085892476e-05, 1.5668829291826114e-05, 1.4448614820139483e-05, 1.0807996659423225e-05, 2.0737868908327073e-05, 2.396255877101794e-05, 1.9252944184700027e-05, 1.644668009248562e-05, 1.497340235800948e-05, 1.2302753020776436e-05, 1.0694708180380985e-05, 1.2048292774125002e-05, 2.090916495944839e-05, 1.6669335309416056e-05, 1.4016508430358954e-05, 1.9095034076599404e-05, 2.868746560125146e-05, 3.0043976948945783e-05, 3.644494427135214e-05, 5.511490599019453e-05, 9.899793076328933e-05, 7.606695726281032e-05, 5.365301694837399e-05, 2.7740055884351023e-05, 3.3802851248765364e-05, 1.8321625248063356e-05, 1.0528205166338012e-05, 1.236709431395866e-05, 7.536691555287689e-06, 5.1510178309399635e-06, 5.647463694913313e-06 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 7.642002219654387e-05, "first_to_last_ratio": 1.1203608181269622, "imbalance_abs_log_ratio": 0.11365079246685211, "last_quartile_mean": 6.821018814662239e-05, "mean": 0.00011010971388714097, "normalized": [ 0.5163109854936148, 0.8673817094155928, 0.5924673517780332, 0.5654643004921971, 0.34399113467095715, 0.7609201839652954, 0.9076117604661024, 0.9981345295043799, 0.6327009050241252, 0.5203303165972624, 0.4437849422865529, 0.41423566265400885, 0.39884159677379066, 0.7686876572920235, 0.6339609024838357, 0.5703063832410542, 0.781268072457197, 1.1797447140196398, 1.2760717444836593, 1.5599326259342914, 2.6126342040159805, 4.419910912775975, 2.9287458133972417, 2.3507635425478965, 1.1736586910474214, 1.2201215929679614, 0.7838637940961916, 0.4869646019277815, 0.46617791476107356, 0.26920984270675297, 0.25672408594104645, 0.2990775247810641 ], "peak_layer": 22, "peak_normalized": 4.419910912775975, "population_cv": 0.8912016880930539, "rest_mean": 7.550380838370916e-05, "spike_contrast": 3.9333327677514873, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0002969816036056727, "top_five_layers": [ 22, 23, 21, 24, 20 ], "top_five_spike_overlap": 4 }, "values": [ 5.685085488948971e-05, 9.550715185469016e-05, 6.523641059175134e-05, 6.226311234058812e-05, 3.787676541833207e-05, 8.378470374736935e-05, 9.993687126552686e-05, 0.00010990430746460333, 6.966651562834159e-05, 5.729342228733003e-05, 4.886503302259371e-05, 4.561137029668316e-05, 4.3916334107052535e-05, 8.463997801300138e-05, 6.980525358812883e-05, 6.279627268668264e-05, 8.602520392742008e-05, 0.00012990135292056948, 0.0001405078946845606, 0.00017176373512484133, 0.0002876764046959579, 0.0004866751260124147, 0.0003224833635613322, 0.0002588419010862708, 0.00012923122267238796, 0.0001343472395092249, 8.631101809442043e-05, 5.3619532991433516e-05, 5.13307168148458e-05, 2.9642618756042793e-05, 2.8267815650906414e-05, 3.293134068371728e-05 ] } } }, "pre_mlp_input": { "all_gradients_finite": true, "all_gradients_present": true, "capture": { "activation_sha256": "e53fa425d89dfeb0e9395eca9bcd73dc0a55314abf45c5b10b43e2cbfa724345", "count": 32, "dtypes": [ "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32", "torch.float32" ], "output_rms_by_layer": [ 0.059514161199331284, 0.08157797157764435, 0.14292451739311218, 0.10997853428125381, 0.0991024449467659, 0.1065012738108635, 0.12155740708112717, 0.1095810979604721, 0.06964563578367233, 0.08311934769153595, 0.09363401681184769, 0.0941167026758194, 0.08564268052577972, 0.09493856877088547, 0.09063401818275452, 0.09473779052495956, 0.10505300015211105, 0.09355910867452621, 0.09618847072124481, 0.11416846513748169, 0.047205954790115356, 0.053352247923612595, 0.06512638926506042, 0.07829347252845764, 0.05368548259139061, 0.05310077592730522, 0.061235878616571426, 0.07320034503936768, 0.05304349213838577, 0.0639304369688034, 0.07404901832342148, 0.08157160878181458 ], "shape": [ 16, 256, 192 ], "storage_unique": true }, "reductions": { "batch_mean_rms": { "statistics": { "first_quartile_mean": 4.6955231027823174e-06, "first_to_last_ratio": 0.14308275980953641, "imbalance_abs_log_ratio": 1.9443320762007668, "last_quartile_mean": 3.2816833481774665e-05, "mean": 1.7864957968072304e-05, "normalized": [ 0.5367890577532618, 0.2021050510263675, 0.1411976282823933, 0.2660218188543664, 0.19890690834720978, 0.16151559172232732, 0.2167464101659875, 0.3793916565295996, 0.424775345287454, 0.3093373886676644, 0.2783613720705816, 0.2825637413001494, 0.3733172205433837, 0.3886765425479794, 0.44060863402002287, 0.44323494933287805, 0.3667341230574315, 0.45728447370296305, 0.42556973592368313, 0.5658685710706646, 2.6227066493164064, 3.0085127149589037, 2.58733070947387, 2.226933740479959, 2.8743975419651484, 2.84666799381699, 2.39504866979417, 1.5908588860103314, 1.8185955715005, 1.4035945808786974, 0.9127606447015644, 0.8535860768970909 ], "peak_layer": 22, "peak_normalized": 3.0085127149589037, "population_cv": 0.9632431009960319, "rest_mean": 1.2359982755933069e-05, "spike_contrast": 3.850476578600324, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 4.759182411362417e-05, "top_five_layers": [ 22, 25, 26, 21, 23 ], "top_five_spike_overlap": 4 }, "values": [ 9.589713954483159e-06, 3.6105982417211635e-06, 2.5224896944564534e-06, 4.7524686124233995e-06, 3.5534635571821127e-06, 2.8854692573077045e-06, 3.872165507345926e-06, 6.777815997338621e-06, 7.588593689433765e-06, 5.526299446501071e-06, 4.972914211975876e-06, 5.047989361628424e-06, 6.669296453765128e-06, 6.9436900957953185e-06, 7.871454727137461e-06, 7.918373739812523e-06, 6.5516896938788705e-06, 8.1693679021555e-06, 7.6027854447602294e-06, 1.0109218237630557e-05, 4.685454405262135e-05, 5.3746953199151903e-05, 4.622255437425338e-05, 3.97840776713565e-05, 5.135099127073772e-05, 5.085560405859724e-05, 4.278744381736033e-05, 2.8420627131708898e-05, 3.248913344577886e-05, 2.507515819161199e-05, 1.6306430552504025e-05, 1.5249279385898262e-05 ] }, "element_rms": { "statistics": { "first_quartile_mean": 1.8878105947806034e-05, "first_to_last_ratio": 0.1437388462831867, "imbalance_abs_log_ratio": 1.939757193388504, "last_quartile_mean": 0.00013133614493199275, "mean": 7.144442986373178e-05, "normalized": [ 0.543944422065902, 0.20498362677674017, 0.14117605251679072, 0.2675823635170904, 0.19429426405696296, 0.15941273479469137, 0.21634544427316113, 0.386139638870723, 0.42865458135412043, 0.3129364237141443, 0.2788876084512782, 0.2815826978190512, 0.3736543391719335, 0.3909795972451435, 0.4389936626557141, 0.44232523153890874, 0.3688366584716598, 0.45947150441297413, 0.42734480687548726, 0.5800824758005463, 2.6187902433353907, 2.987231504916195, 2.580721746390286, 2.2092452217225382, 2.879951595578818, 2.8406783220158585, 2.397028447242086, 1.5906421267626043, 1.8247593500794728, 1.4018234858691145, 0.9198949147470885, 0.8516049069575247 ], "peak_layer": 22, "peak_normalized": 2.987231504916195, "population_cv": 0.9601858610516129, "rest_mean": 4.954554700547036e-05, "spike_contrast": 3.828767846228416, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00018969839729834347, "top_five_layers": [ 22, 25, 26, 21, 23 ], "top_five_spike_overlap": 4 }, "values": [ 3.886179911205545e-05, 1.4644938346464187e-05, 1.008624258247437e-05, 1.911726940306835e-05, 1.3881242921343073e-05, 1.1389151950425003e-05, 1.5456676919711754e-05, 2.7587526346906088e-05, 3.062498217332177e-05, 2.2357564375852235e-05, 1.9924966181861237e-05, 2.0117515305173583e-05, 2.6695521228248253e-05, 2.7933314413530752e-05, 3.13636519422289e-05, 3.160167398164049e-05, 2.6351324777351692e-05, 3.282667967141606e-05, 3.0531406082445756e-05, 4.144366175751202e-05, 0.00018709797586780041, 0.00021342105173971504, 0.0001843781938077882, 0.00015783826529514045, 0.0002057564997812733, 0.0002029506431426853, 0.00017125433078035712, 0.00011364251986378804, 0.00013036889140494168, 0.00010015247971750796, 6.572136771865189e-05, 6.08424270467367e-05 ] }, "global_l2": { "statistics": { "first_quartile_mean": 0.016741293016821146, "first_to_last_ratio": 0.14373884545887533, "imbalance_abs_log_ratio": 1.9397571991232891, "last_quartile_mean": 0.11647020652890205, "mean": 0.06335763493552804, "normalized": [ 0.5439444372437541, 0.20498362168576487, 0.14117604516877305, 0.2675823445899417, 0.1942942529879486, 0.15941273039509152, 0.2163454315588509, 0.38613966530889043, 0.4286545697273512, 0.3129364189494029, 0.278887605667477, 0.2815826915705289, 0.3736543321474427, 0.3909795777927573, 0.43899363576159356, 0.44232520797796643, 0.3688366746353626, 0.45947150248290297, 0.4273447896704886, 0.580082454376481, 2.6187903327724635, 2.987231533325295, 2.5807218070831297, 2.2092452282911403, 2.8799516329199686, 2.84067822247012, 2.397028505583422, 1.5906421518499794, 1.8247592808395534, 1.4018234450909468, 0.9198949415397243, 0.8516049285354858 ], "peak_layer": 22, "peak_normalized": 2.987231533325295, "population_cv": 0.9601858709388058, "rest_mean": 0.04393748607900408, "spike_contrast": 3.8287679558695995, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.16822643876075744, "top_five_layers": [ 22, 25, 26, 21, 23 ], "top_five_spike_overlap": 4 }, "values": [ 0.03446303308010101, 0.01298727747052908, 0.00894458033144474, 0.01695338450372219, 0.012310024350881577, 0.010100013576447964, 0.013707134872674942, 0.024464895948767662, 0.02715853974223137, 0.01982691138982773, 0.017669659107923508, 0.017840413376688957, 0.023673854768276215, 0.024771541357040405, 0.02781359851360321, 0.02802467904984951, 0.02336861938238144, 0.029111027717590332, 0.02707555517554283, 0.03675265237689018, 0.16592036187648773, 0.18926392495632172, 0.16350843012332916, 0.13997255265712738, 0.18246692419052124, 0.1799786537885666, 0.15187005698680878, 0.10077932476997375, 0.11561243236064911, 0.08881621807813644, 0.05828236788511276, 0.053955674171447754 ] }, "token_mean_rms": { "statistics": { "first_quartile_mean": 1.0825735472508313e-06, "first_to_last_ratio": 0.12187576812955331, "imbalance_abs_log_ratio": 2.104753047082524, "last_quartile_mean": 8.882598763193528e-06, "mean": 4.656443639561303e-06, "normalized": [ 0.5462983207096154, 0.1799290732737773, 0.13161762512632208, 0.2712494331106157, 0.1613911681124062, 0.13237661267085615, 0.15721616286614848, 0.2798363906909718, 0.34662434040104295, 0.2682122377700395, 0.24445710198520895, 0.24681111820039023, 0.3364691861798734, 0.3619231630019147, 0.42744951901080175, 0.43112099033231693, 0.36249737820633765, 0.4368387762566429, 0.42578881412518377, 0.38711372693052853, 2.644134376023103, 3.0274176414789977, 2.680305221816438, 2.2521785297578925, 2.965193895729998, 2.901136489590974, 2.603841896063781, 1.6862657755052308, 1.872923926339135, 1.3894795988927409, 0.9615945171774488, 0.8803069926632656 ], "peak_layer": 22, "peak_normalized": 3.0274176414789977, "population_cv": 1.0082602587575777, "rest_mean": 3.1785867888713795e-06, "spike_contrast": 3.9756254815914156, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 1.2636870633286889e-05, "top_five_layers": [ 22, 25, 26, 23, 21 ], "top_five_spike_overlap": 4 }, "values": [ 2.543807340771309e-06, 8.3782958881784e-07, 6.128700533736264e-07, 1.2630576975425356e-06, 7.515088782383827e-07, 6.164042360978783e-07, 7.32068201614311e-07, 1.3030423815507675e-06, 1.6140367051775684e-06, 1.2489151686168043e-06, 1.138300717684615e-06, 1.14926206151722e-06, 1.5667498018956394e-06, 1.6852748103701742e-06, 1.990394594031386e-06, 2.0074905933142873e-06, 1.687948611106549e-06, 2.034115141213988e-06, 1.9826616153295618e-06, 1.8025732515525306e-06, 1.2312262697378173e-05, 1.409699962096056e-05, 1.2480690202210099e-05, 1.0487142390047666e-05, 1.3807258255837951e-05, 1.3508978554455098e-05, 1.2124643035349436e-05, 7.85200154496124e-06, 8.721164704184048e-06, 6.470033440564293e-06, 4.477610673347954e-06, 4.099099896848202e-06 ] }, "token_rms_mean": { "statistics": { "first_quartile_mean": 1.2557812397062662e-05, "first_to_last_ratio": 0.15959566659574662, "imbalance_abs_log_ratio": 1.8351117459907644, "last_quartile_mean": 7.868517150200205e-05, "mean": 4.5438396199415365e-05, "normalized": [ 0.5876869810338485, 0.2186593308079107, 0.115659623677568, 0.24728992953724943, 0.2138823134903866, 0.17309343907534852, 0.22828222046505, 0.4264066618299725, 0.47941738273017553, 0.34947398487973863, 0.31642185290041636, 0.3326608895247323, 0.42753132054545495, 0.43329608717407087, 0.4807098955780001, 0.491094994316893, 0.41296023678331367, 0.5280992844682807, 0.4827771475203192, 0.6138123378227869, 2.7117823469683264, 3.061642080099734, 2.646250140140893, 2.167597408443836, 2.9446942269972274, 2.694165709502317, 2.239316799454017, 1.5139318638986488, 1.6066586018929343, 1.2634426578175135, 0.8290147862338136, 0.7622874643892231 ], "peak_layer": 22, "peak_normalized": 3.061642080099734, "population_cv": 0.9412918554701797, "rest_mean": 3.107991987808082e-05, "spike_contrast": 3.9567080229621103, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.00012297416833462194, "top_five_layers": [ 22, 25, 21, 26, 23 ], "top_five_spike_overlap": 4 }, "values": [ 2.6703553885454312e-05, 9.935529305948876e-06, 5.255387804936618e-06, 1.1236457794439048e-05, 9.718469300423749e-06, 7.86508826422505e-06, 1.0372777978773229e-05, 1.9375234842300415e-05, 2.178395698138047e-05, 1.587953738635406e-05, 1.4377701518242247e-05, 1.511557729827473e-05, 1.9426337530603632e-05, 1.968827928067185e-05, 2.1842686692252755e-05, 2.231456892332062e-05, 1.876425085356459e-05, 2.3995984520297498e-05, 2.1936619305051863e-05, 2.789064819808118e-05, 0.00012321904068812728, 0.0001391161058563739, 0.00012024136231048033, 9.849214984569699e-05, 0.00013380218297243118, 0.00012241856893524528, 0.00010175096394959837, 6.879063585074618e-05, 7.300399011000991e-05, 5.740880806115456e-05, 3.766910231206566e-05, 3.463711982476525e-05 ] }, "token_rms_median": { "statistics": { "first_quartile_mean": 8.367274347165221e-06, "first_to_last_ratio": 0.1920014554991945, "imbalance_abs_log_ratio": 1.6502523262581177, "last_quartile_mean": 4.3579223529377487e-05, "mean": 2.787530080183842e-05, "normalized": [ 0.6522901960458961, 0.23732362933486428, 0.10383521258256681, 0.24073339013803968, 0.24132146388193534, 0.19483084771733855, 0.2475207738927628, 0.483488789493259, 0.550030749638162, 0.4005903921146771, 0.3626794403805512, 0.398664404925254, 0.5057551655640742, 0.49633936232785464, 0.5444163811542189, 0.5645330449101175, 0.481087325236049, 0.6303807295780811, 0.5534160889368265, 0.6675682370643486, 2.937265214456223, 3.231070824381384, 2.7157464748293116, 2.0522050942602807, 2.8791759008165254, 2.406039643209466, 1.9404796965168836, 1.3618287054639213, 1.3825321378732942, 1.123237294980051, 0.7355691904186271, 0.6780441978771551 ], "peak_layer": 22, "peak_normalized": 3.231070824381384, "population_cv": 0.9137826743997257, "rest_mean": 1.8774052764288002e-05, "spike_contrast": 4.1025792976954865, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 7.702204020461067e-05, "top_five_layers": [ 22, 21, 25, 23, 26 ], "top_five_spike_overlap": 4 }, "values": [ 1.8182785424869508e-05, 6.615467555093346e-06, 2.8944377845618874e-06, 6.710515663144179e-06, 6.726908395648934e-06, 5.430968485597987e-06, 6.899716026964597e-06, 1.3477395441441331e-05, 1.5332272596424446e-05, 1.1166577678523026e-05, 1.0109798495250288e-05, 1.111289020627737e-05, 1.409807737218216e-05, 1.3835609024681617e-05, 1.5175770386122167e-05, 1.5736528439447284e-05, 1.3410453902906738e-05, 1.7572052456671372e-05, 1.5426639947691e-05, 1.86086654139217e-05, 8.187715138774365e-05, 9.006707114167511e-05, 7.570224988739938e-05, 5.720583430957049e-05, 8.025789429666474e-05, 6.706907879561186e-05, 5.409145524026826e-05, 3.796138480538502e-05, 3.8538499211426824e-05, 3.131057746941224e-05, 2.0504212443483993e-05, 1.890068597276695e-05 ] }, "token_rms_p95": { "statistics": { "first_quartile_mean": 3.742885974133969e-05, "first_to_last_ratio": 0.13285814939480156, "imbalance_abs_log_ratio": 2.018473265828943, "last_quartile_mean": 0.00028172046586405486, "mean": 0.00014909372896454443, "normalized": [ 0.5351570886371335, 0.1998816464903299, 0.1134003859562838, 0.2373819769757024, 0.18971959520637977, 0.1492636791238371, 0.21004413698536548, 0.3734913450927867, 0.42927632587153325, 0.3024206473119055, 0.27462175768495817, 0.27240928064339276, 0.3512855646324889, 0.3655094888356237, 0.41973324501001746, 0.42402062842094873, 0.360798486332806, 0.4559662563090299, 0.41985007515873884, 0.5597866310716989, 2.4844541141001115, 2.96085466615825, 2.5443412288142064, 2.249909733543605, 2.9984626521607822, 2.895026684700318, 2.601768175835175, 1.7136369023458127, 1.8313353225818492, 1.363588302533558, 0.8854239763588411, 0.82717999911653 ], "peak_layer": 25, "peak_normalized": 2.9984626521607822, "population_cv": 0.9800748961570518, "rest_mean": 0.00010360345199673989, "spike_contrast": 3.810116525877056, "spike_layers": [ 21, 22, 23, 24, 25 ], "spike_mean": 0.0003947412245906889, "top_five_layers": [ 25, 22, 26, 27, 23 ], "top_five_spike_overlap": 3 }, "values": [ 7.978856592671946e-05, 2.980110002681613e-05, 1.6907286408240907e-05, 3.53921641362831e-05, 2.8286001906963065e-05, 2.2254278519540094e-05, 3.131626363028772e-05, 5.568521737586707e-05, 6.400240818038583e-05, 4.508902202360332e-05, 4.0944381908047944e-05, 4.061451545567252e-05, 5.2374474762473255e-05, 5.449517266242765e-05, 6.257959466893226e-05, 6.321881664916873e-05, 5.379279173212126e-05, 6.79817094351165e-05, 6.259701331146061e-05, 8.34606762509793e-05, 0.0003704165283124894, 0.00044144486309960485, 0.0003793453215621412, 0.0003354474320076406, 0.0004470519779715687, 0.0004316303238738328, 0.00038790731923654675, 0.00025549251586198807, 0.00027304061222821474, 0.00020330246479716152, 0.00013201116234995425, 0.00012332735059317201 ] } } } } }, "parent_learned_round06_exact": true, "step": 8000 } ], "environment": { "autocast": "cuda-bfloat16-forward-fp32-cross-entropy", "compile": false, "compute_capability": [ 12, 0 ], "cublas_workspace_config": ":4096:8", "cuda": "12.8", "deterministic_algorithms": true, "gpu": "NVIDIA GeForce RTX 5090", "python": "3.10.14", "torch": "2.11.0+cu128" }, "evaluations": [ { "bits_per_byte": 7.9557801867581395, "cross_entropy_nats": 5.514526605606079, "step": 0 }, { "bits_per_byte": 5.165135820292823, "cross_entropy_nats": 3.5801993310451508, "step": 100 }, { "bits_per_byte": 3.109974443324717, "cross_entropy_nats": 2.155670017004013, "step": 500 }, { "bits_per_byte": 1.9592225965086283, "cross_entropy_nats": 1.358029618859291, "step": 2000 }, { "bits_per_byte": 1.8027438806514628, "cross_entropy_nats": 1.249566838145256, "step": 4000 }, { "bits_per_byte": 1.7123525940760473, "cross_entropy_nats": 1.1869123727083206, "step": 8000 } ], "hashes": { "final_mixer_parameters": "f16f089e57699cb35addeee1608f1119edec41037e2b24b645f889b4f23c5f71", "final_model_state": "3f0b97ece3a15571ba3d656f589f512ca0bb9e20083c9f58a42ccaee14892f59", "final_optimizer_state": "ed03e6fbd4a12d8b063dcb22e0437754285f54d585374cd52fbd534f05d24637", "final_public_parameters": "1fbfd02c38aae6a81ba07a28d1135ce7d290651ca00bf5fa93595a997937551e", "initial_mixer_parameters": "c5a06c218c4501b16fcccee115aa8c79d5dceade78b38bbb547e4f5b2202516d", "initial_public_parameter_elements": 18985152, "initial_public_parameter_structure": "7625a2d62805070dfbe720e9243f103d23c47e27d3c14de1af55fd6187e81670", "initial_public_parameter_tensors": 227, "initial_public_parameters": "73bbe569e1a47981386ebaf59ca891174746bf81897fe5403676ad6d50423c58" }, "manifest": { "data_file_sha256": "080afb17d1e036c0bba0a799fdb8b98ee4ad652bd42dd1b3b67110dd2ede6371", "data_path": "experiments/k3/attnres_gradient/manifest.json", "diagnostic_tensor_sha256": "21117e31db302b10d67b63f035665dc8f220b879d216ccd12b7d2ba86e7b1716", "file_sha256": "db01e92ef2cf0896212fcd529429bd94a344de0e1195db7f87b9a56dc3449139", "formal_schedule_sha256": "5041e09b167f229248d2462324e8c254b8f5938975f135dcd8192b00a54a4f4e", "input_gate_tensor_hashes": { "0": "65136111a29a042e61a7909132560d95cd4bcf0f9b52f64d0fb2e57773856434", "1": "d995676b4e7dec8f661cd8c2345fe7fc7a513c17f528c02fc946a441a6995a94", "7999": "2345e7ac3decca2bdaebf13094fdc92fcabef42e3e461a2fefd6e8e7e76baccc" }, "parent_file_sha256": "d5302a249249a07d362819134763d14e7d32307f22cff416c665ed9606142fef", "parent_path": "experiments/k3/attnres_spike/manifest.json", "path": "experiments/k3/attnres_local_path/manifest.json", "selector_contract_sha256": "187a564bf95892b56c465e44f66753bee2246c98cc681aeb715f83b07669f128", "validation_tensor_sha256": "f459316f13078a163b47c133511bb7181e05170ab89516e196490113893ce338" }, "model": { "aggregation_groups": 8, "blocks_per_group": 4, "d_ff": 768, "d_model": 192, "heads": 6, "layers": 32, "parameters": { "core": 18985152, "embedding": 98304, "mixer": 24960, "total": 19010112 } }, "parent_protocol_id": "llm-atlas-k3-attnres-spike-path-v1", "protocol_id": "llm-atlas-k3-attnres-local-path-v1", "round06_equivalence": { "checks": { "evaluations": true, "final_model_state": true, "final_optimizer_state": true, "parent_learned_diagnostics": true, "round06_endpoints": true, "training_history": true }, "passed": true }, "run_kind": "formal", "schema_version": 1, "seed": 2026073001, "steps": 8000, "target_bytes_seen": 65536000, "timing": { "mean_ms": 228.44162397260223, "measured_steps": 7980, "median_ms": 228.36488900065888, "p95_ms": 231.9016571044922, "peak_allocated_bytes": 12841009152, "peak_reserved_bytes": 13922992128, "warmup_steps_excluded": 20 }, "training_history": [ { "bits_per_byte": 7.958777844362678, "learning_rate": 7.499999999999999e-07, "loss_nats": 5.516604423522949, "step": 1, "unclipped_grad_norm": 21.323171615600586 }, { "bits_per_byte": 7.247870358756172, "learning_rate": 7.499999999999999e-06, "loss_nats": 5.02384090423584, "step": 10, "unclipped_grad_norm": 11.153014183044434 }, { "bits_per_byte": 6.7336346654397365, "learning_rate": 1.4999999999999999e-05, "loss_nats": 4.667399883270264, "step": 20, "unclipped_grad_norm": 3.4888553619384766 }, { "bits_per_byte": 6.611855126224988, "learning_rate": 2.2499999999999998e-05, "loss_nats": 4.582988739013672, "step": 30, "unclipped_grad_norm": 3.075686454772949 }, { "bits_per_byte": 6.3915955694037825, "learning_rate": 2.9999999999999997e-05, "loss_nats": 4.43031644821167, "step": 40, "unclipped_grad_norm": 3.2318925857543945 }, { "bits_per_byte": 6.093941755902502, "learning_rate": 3.75e-05, "loss_nats": 4.223998546600342, "step": 50, "unclipped_grad_norm": 2.8704335689544678 }, { "bits_per_byte": 5.8858000966610895, "learning_rate": 4.4999999999999996e-05, "loss_nats": 4.079725742340088, "step": 60, "unclipped_grad_norm": 2.669306755065918 }, { "bits_per_byte": 5.715495996470702, "learning_rate": 5.2499999999999995e-05, "loss_nats": 3.9616799354553223, "step": 70, "unclipped_grad_norm": 3.1737496852874756 }, { "bits_per_byte": 5.565897919513593, "learning_rate": 5.9999999999999995e-05, "loss_nats": 3.8579864501953125, "step": 80, "unclipped_grad_norm": 2.5544967651367188 }, { "bits_per_byte": 5.354669171689194, "learning_rate": 6.749999999999999e-05, "loss_nats": 3.711573839187622, "step": 90, "unclipped_grad_norm": 2.425950050354004 }, { "bits_per_byte": 5.180876231443403, "learning_rate": 7.5e-05, "loss_nats": 3.5911097526550293, "step": 100, "unclipped_grad_norm": 2.2783501148223877 }, { "bits_per_byte": 4.992272489270609, "learning_rate": 8.249999999999999e-05, "loss_nats": 3.4603796005249023, "step": 110, "unclipped_grad_norm": 2.1105899810791016 }, { "bits_per_byte": 4.775783480086919, "learning_rate": 8.999999999999999e-05, "loss_nats": 3.3103208541870117, "step": 120, "unclipped_grad_norm": 1.8969556093215942 }, { "bits_per_byte": 4.582375918510225, "learning_rate": 9.75e-05, "loss_nats": 3.1762609481811523, "step": 130, "unclipped_grad_norm": 1.7015109062194824 }, { "bits_per_byte": 4.427903165297061, "learning_rate": 0.00010499999999999999, "loss_nats": 3.0691885948181152, "step": 140, "unclipped_grad_norm": 1.446854829788208 }, { "bits_per_byte": 4.271120341117507, "learning_rate": 0.0001125, "loss_nats": 2.960515022277832, "step": 150, "unclipped_grad_norm": 1.2307006120681763 }, { "bits_per_byte": 4.166711049639903, "learning_rate": 0.00011999999999999999, "loss_nats": 2.888144016265869, "step": 160, "unclipped_grad_norm": 1.0598198175430298 }, { "bits_per_byte": 4.026318516052877, "learning_rate": 0.00012749999999999998, "loss_nats": 2.7908313274383545, "step": 170, "unclipped_grad_norm": 0.9654788374900818 }, { "bits_per_byte": 3.94755837315965, "learning_rate": 0.00013499999999999997, "loss_nats": 2.736238956451416, "step": 180, "unclipped_grad_norm": 0.8419675230979919 }, { "bits_per_byte": 3.838975750676209, "learning_rate": 0.0001425, "loss_nats": 2.660975217819214, "step": 190, "unclipped_grad_norm": 0.817121684551239 }, { "bits_per_byte": 3.774983758012572, "learning_rate": 0.00015, "loss_nats": 2.616619348526001, "step": 200, "unclipped_grad_norm": 0.955824613571167 }, { "bits_per_byte": 3.7344016998190783, "learning_rate": 0.0001575, "loss_nats": 2.5884900093078613, "step": 210, "unclipped_grad_norm": 0.8557147979736328 }, { "bits_per_byte": 3.63166201591605, "learning_rate": 0.00016499999999999997, "loss_nats": 2.5172762870788574, "step": 220, "unclipped_grad_norm": 0.6891153454780579 }, { "bits_per_byte": 3.6208855830129867, "learning_rate": 0.0001725, "loss_nats": 2.5098066329956055, "step": 230, "unclipped_grad_norm": 1.036860466003418 }, { "bits_per_byte": 3.4987607026353063, "learning_rate": 0.00017999999999999998, "loss_nats": 2.4251561164855957, "step": 240, "unclipped_grad_norm": 0.6406477689743042 }, { "bits_per_byte": 3.5387748741156506, "learning_rate": 0.0001875, "loss_nats": 2.4528918266296387, "step": 250, "unclipped_grad_norm": 1.106186866760254 }, { "bits_per_byte": 3.5260464381252943, "learning_rate": 0.000195, "loss_nats": 2.4440691471099854, "step": 260, "unclipped_grad_norm": 0.8556521534919739 }, { "bits_per_byte": 3.5219934949750615, "learning_rate": 0.00020249999999999996, "loss_nats": 2.4412598609924316, "step": 270, "unclipped_grad_norm": 0.9917709827423096 }, { "bits_per_byte": 3.474740573719901, "learning_rate": 0.00020999999999999998, "loss_nats": 2.4085066318511963, "step": 280, "unclipped_grad_norm": 1.2901222705841064 }, { "bits_per_byte": 3.363479149466621, "learning_rate": 0.00021749999999999997, "loss_nats": 2.331386089324951, "step": 290, "unclipped_grad_norm": 0.7895627617835999 }, { "bits_per_byte": 3.4037392561376993, "learning_rate": 0.000225, "loss_nats": 2.3592922687530518, "step": 300, "unclipped_grad_norm": 1.5281883478164673 }, { "bits_per_byte": 3.3541442751430894, "learning_rate": 0.00023249999999999996, "loss_nats": 2.324915647506714, "step": 310, "unclipped_grad_norm": 0.8648361563682556 }, { "bits_per_byte": 3.3434108379027414, "learning_rate": 0.00023999999999999998, "loss_nats": 2.3174757957458496, "step": 320, "unclipped_grad_norm": 1.4613573551177979 }, { "bits_per_byte": 3.3325662998699364, "learning_rate": 0.0002475, "loss_nats": 2.3099589347839355, "step": 330, "unclipped_grad_norm": 1.1215400695800781 }, { "bits_per_byte": 3.4180712583647006, "learning_rate": 0.00025499999999999996, "loss_nats": 2.3692264556884766, "step": 340, "unclipped_grad_norm": 1.1166281700134277 }, { "bits_per_byte": 3.3577222022116393, "learning_rate": 0.0002625, "loss_nats": 2.3273956775665283, "step": 350, "unclipped_grad_norm": 1.0810235738754272 }, { "bits_per_byte": 3.2923828974611427, "learning_rate": 0.00026999999999999995, "loss_nats": 2.2821059226989746, "step": 360, "unclipped_grad_norm": 1.128420114517212 }, { "bits_per_byte": 3.291989401155969, "learning_rate": 0.00027749999999999997, "loss_nats": 2.2818331718444824, "step": 370, "unclipped_grad_norm": 1.0846341848373413 }, { "bits_per_byte": 3.3389695579268603, "learning_rate": 0.000285, "loss_nats": 2.3143973350524902, "step": 380, "unclipped_grad_norm": 1.0868512392044067 }, { "bits_per_byte": 3.256548592327368, "learning_rate": 0.0002925, "loss_nats": 2.257267475128174, "step": 390, "unclipped_grad_norm": 1.1476690769195557 }, { "bits_per_byte": 3.2058776878689605, "learning_rate": 0.0003, "loss_nats": 2.2221450805664062, "step": 400, "unclipped_grad_norm": 1.1263198852539062 }, { "bits_per_byte": 3.195324144481415, "learning_rate": 0.0002999988466112388, "loss_nats": 2.214829921722412, "step": 410, "unclipped_grad_norm": 1.1377208232879639 }, { "bits_per_byte": 3.227808915506986, "learning_rate": 0.00029999538646466355, "loss_nats": 2.237346649169922, "step": 420, "unclipped_grad_norm": 1.2929860353469849 }, { "bits_per_byte": 3.1876103786249166, "learning_rate": 0.0002999896196193985, "loss_nats": 2.2094831466674805, "step": 430, "unclipped_grad_norm": 1.071694016456604 }, { "bits_per_byte": 3.2374495749837515, "learning_rate": 0.00029998154617398326, "loss_nats": 2.2440290451049805, "step": 440, "unclipped_grad_norm": 0.9089367985725403 }, { "bits_per_byte": 3.155426921203657, "learning_rate": 0.0002999711662663707, "loss_nats": 2.1871752738952637, "step": 450, "unclipped_grad_norm": 1.1129212379455566 }, { "bits_per_byte": 3.188086770567806, "learning_rate": 0.0002999584800739247, "loss_nats": 2.209813356399536, "step": 460, "unclipped_grad_norm": 1.3905333280563354 }, { "bits_per_byte": 3.128408790718412, "learning_rate": 0.0002999434878134173, "loss_nats": 2.168447732925415, "step": 470, "unclipped_grad_norm": 1.059533953666687 }, { "bits_per_byte": 3.136691131218663, "learning_rate": 0.000299926189741025, "loss_nats": 2.1741886138916016, "step": 480, "unclipped_grad_norm": 1.0161854028701782 }, { "bits_per_byte": 3.0887341129908705, "learning_rate": 0.000299906586152324, "loss_nats": 2.1409473419189453, "step": 490, "unclipped_grad_norm": 1.385905385017395 }, { "bits_per_byte": 3.033835874974255, "learning_rate": 0.0002998846773822856, "loss_nats": 2.1028947830200195, "step": 500, "unclipped_grad_norm": 1.1716443300247192 }, { "bits_per_byte": 3.0607071322756636, "learning_rate": 0.0002998604638052702, "loss_nats": 2.121520519256592, "step": 510, "unclipped_grad_norm": 0.947066605091095 }, { "bits_per_byte": 2.997100036784657, "learning_rate": 0.0002998339458350211, "loss_nats": 2.0774314403533936, "step": 520, "unclipped_grad_norm": 1.352820634841919 }, { "bits_per_byte": 3.0643050093317137, "learning_rate": 0.0002998051239246572, "loss_nats": 2.124014377593994, "step": 530, "unclipped_grad_norm": 1.2646857500076294 }, { "bits_per_byte": 3.008989197438803, "learning_rate": 0.00029977399856666537, "loss_nats": 2.085672378540039, "step": 540, "unclipped_grad_norm": 1.0451104640960693 }, { "bits_per_byte": 2.978105584892765, "learning_rate": 0.0002997405702928922, "loss_nats": 2.064265489578247, "step": 550, "unclipped_grad_norm": 1.2619587182998657 }, { "bits_per_byte": 2.968300166036473, "learning_rate": 0.0002997048396745345, "loss_nats": 2.057468891143799, "step": 560, "unclipped_grad_norm": 0.9861598610877991 }, { "bits_per_byte": 2.9221661639078262, "learning_rate": 0.0002996668073221302, "loss_nats": 2.025491237640381, "step": 570, "unclipped_grad_norm": 1.1261875629425049 }, { "bits_per_byte": 2.96999178739036, "learning_rate": 0.000299626473885547, "loss_nats": 2.0586414337158203, "step": 580, "unclipped_grad_norm": 0.8640331625938416 }, { "bits_per_byte": 2.883698804389342, "learning_rate": 0.00029958384005397226, "loss_nats": 1.9988276958465576, "step": 590, "unclipped_grad_norm": 1.0034456253051758 }, { "bits_per_byte": 2.9404706214044567, "learning_rate": 0.0002995389065559004, "loss_nats": 2.0381789207458496, "step": 600, "unclipped_grad_norm": 0.8714175224304199 }, { "bits_per_byte": 2.8380133330141457, "learning_rate": 0.000299491674159121, "loss_nats": 1.967160940170288, "step": 610, "unclipped_grad_norm": 0.9427422881126404 }, { "bits_per_byte": 2.894644124255553, "learning_rate": 0.00029944214367070535, "loss_nats": 2.0064144134521484, "step": 620, "unclipped_grad_norm": 1.0238139629364014 }, { "bits_per_byte": 2.81883675149499, "learning_rate": 0.00029939031593699284, "loss_nats": 1.9538687467575073, "step": 630, "unclipped_grad_norm": 0.9875481128692627 }, { "bits_per_byte": 2.7787291934352267, "learning_rate": 0.00029933619184357654, "loss_nats": 1.9260683059692383, "step": 640, "unclipped_grad_norm": 1.0125378370285034 }, { "bits_per_byte": 2.728423580127252, "learning_rate": 0.00029927977231528774, "loss_nats": 1.8911991119384766, "step": 650, "unclipped_grad_norm": 1.5507988929748535 }, { "bits_per_byte": 2.649404087396524, "learning_rate": 0.00029922105831618057, "loss_nats": 1.8364269733428955, "step": 660, "unclipped_grad_norm": 0.9040125012397766 }, { "bits_per_byte": 2.7024973674923345, "learning_rate": 0.0002991600508495153, "loss_nats": 1.8732284307479858, "step": 670, "unclipped_grad_norm": 1.205861210823059 }, { "bits_per_byte": 2.641403626461037, "learning_rate": 0.00029909675095774114, "loss_nats": 1.8308814764022827, "step": 680, "unclipped_grad_norm": 1.3370649814605713 }, { "bits_per_byte": 2.7325929555321187, "learning_rate": 0.0002990311597224787, "loss_nats": 1.8940891027450562, "step": 690, "unclipped_grad_norm": 1.6124995946884155 }, { "bits_per_byte": 2.6130706886099455, "learning_rate": 0.00029896327826450103, "loss_nats": 1.8112425804138184, "step": 700, "unclipped_grad_norm": 1.08383309841156 }, { "bits_per_byte": 2.628106443844277, "learning_rate": 0.0002988931077437152, "loss_nats": 1.821664571762085, "step": 710, "unclipped_grad_norm": 1.0767779350280762 }, { "bits_per_byte": 2.6725746220166666, "learning_rate": 0.00029882064935914164, "loss_nats": 1.852487564086914, "step": 720, "unclipped_grad_norm": 1.296352505683899 }, { "bits_per_byte": 2.608218370098502, "learning_rate": 0.0002987459043488943, "loss_nats": 1.8078792095184326, "step": 730, "unclipped_grad_norm": 1.0864766836166382 }, { "bits_per_byte": 2.555699340073923, "learning_rate": 0.0002986688739901591, "loss_nats": 1.7714757919311523, "step": 740, "unclipped_grad_norm": 1.0339913368225098 }, { "bits_per_byte": 2.515936435246489, "learning_rate": 0.0002985895595991723, "loss_nats": 1.743914246559143, "step": 750, "unclipped_grad_norm": 1.1017143726348877 }, { "bits_per_byte": 2.4657125136976736, "learning_rate": 0.00029850796253119824, "loss_nats": 1.709101676940918, "step": 760, "unclipped_grad_norm": 1.7173939943313599 }, { "bits_per_byte": 2.601046349592221, "learning_rate": 0.00029842408418050544, "loss_nats": 1.802907943725586, "step": 770, "unclipped_grad_norm": 0.9487742781639099 }, { "bits_per_byte": 2.4794518637098015, "learning_rate": 0.0002983379259803436, "loss_nats": 1.7186250686645508, "step": 780, "unclipped_grad_norm": 0.9247332811355591 }, { "bits_per_byte": 2.466098098800908, "learning_rate": 0.0002982494894029186, "loss_nats": 1.7093689441680908, "step": 790, "unclipped_grad_norm": 1.9121034145355225 }, { "bits_per_byte": 2.4025570446478333, "learning_rate": 0.0002981587759593675, "loss_nats": 1.66532564163208, "step": 800, "unclipped_grad_norm": 1.1767630577087402 }, { "bits_per_byte": 2.455517898102479, "learning_rate": 0.00029806578719973283, "loss_nats": 1.7020353078842163, "step": 810, "unclipped_grad_norm": 1.1299240589141846 }, { "bits_per_byte": 2.400107839716896, "learning_rate": 0.00029797052471293596, "loss_nats": 1.6636279821395874, "step": 820, "unclipped_grad_norm": 1.0006046295166016 }, { "bits_per_byte": 2.4500607166079513, "learning_rate": 0.0002978729901267499, "loss_nats": 1.6982526779174805, "step": 830, "unclipped_grad_norm": 1.0554603338241577 }, { "bits_per_byte": 2.5153210813217015, "learning_rate": 0.0002977731851077716, "loss_nats": 1.7434877157211304, "step": 840, "unclipped_grad_norm": 1.0736713409423828 }, { "bits_per_byte": 2.399556119372928, "learning_rate": 0.0002976711113613935, "loss_nats": 1.6632455587387085, "step": 850, "unclipped_grad_norm": 0.8279580473899841 }, { "bits_per_byte": 2.414819407654351, "learning_rate": 0.0002975667706317743, "loss_nats": 1.6738252639770508, "step": 860, "unclipped_grad_norm": 1.2342168092727661 }, { "bits_per_byte": 2.3222158652452247, "learning_rate": 0.00029746016470180915, "loss_nats": 1.6096373796463013, "step": 870, "unclipped_grad_norm": 1.0648512840270996 }, { "bits_per_byte": 2.3629336177502487, "learning_rate": 0.00029735129539309926, "loss_nats": 1.6378607749938965, "step": 880, "unclipped_grad_norm": 0.9258757829666138 }, { "bits_per_byte": 2.2961121665146167, "learning_rate": 0.0002972401645659207, "loss_nats": 1.5915436744689941, "step": 890, "unclipped_grad_norm": 1.0717874765396118 }, { "bits_per_byte": 2.3158979105831285, "learning_rate": 0.00029712677411919283, "loss_nats": 1.6052581071853638, "step": 900, "unclipped_grad_norm": 0.9598665833473206 }, { "bits_per_byte": 2.3301258633063577, "learning_rate": 0.0002970111259904454, "loss_nats": 1.6151201725006104, "step": 910, "unclipped_grad_norm": 1.0995051860809326 }, { "bits_per_byte": 2.3123376977276173, "learning_rate": 0.0002968932221557859, "loss_nats": 1.602790355682373, "step": 920, "unclipped_grad_norm": 1.1837356090545654 }, { "bits_per_byte": 2.3340250539667196, "learning_rate": 0.0002967730646298655, "loss_nats": 1.6178228855133057, "step": 930, "unclipped_grad_norm": 1.1502224206924438 }, { "bits_per_byte": 2.330418921743428, "learning_rate": 0.0002966506554658449, "loss_nats": 1.6153233051300049, "step": 940, "unclipped_grad_norm": 1.0088714361190796 }, { "bits_per_byte": 2.2995397807463123, "learning_rate": 0.00029652599675535876, "loss_nats": 1.5939195156097412, "step": 950, "unclipped_grad_norm": 0.9483612179756165 }, { "bits_per_byte": 2.3536873144919115, "learning_rate": 0.0002963990906284808, "loss_nats": 1.6314517259597778, "step": 960, "unclipped_grad_norm": 0.9582105278968811 }, { "bits_per_byte": 2.2526236015215417, "learning_rate": 0.0002962699392536863, "loss_nats": 1.5613996982574463, "step": 970, "unclipped_grad_norm": 1.2002545595169067 }, { "bits_per_byte": 2.3230659754884395, "learning_rate": 0.0002961385448378161, "loss_nats": 1.6102266311645508, "step": 980, "unclipped_grad_norm": 0.9191399812698364 }, { "bits_per_byte": 2.285470911975131, "learning_rate": 0.00029600490962603817, "loss_nats": 1.584167718887329, "step": 990, "unclipped_grad_norm": 1.053741216659546 }, { "bits_per_byte": 2.248061417742104, "learning_rate": 0.00029586903590180956, "loss_nats": 1.5582374334335327, "step": 1000, "unclipped_grad_norm": 0.9024814963340759 }, { "bits_per_byte": 2.2402749032242903, "learning_rate": 0.00029573092598683735, "loss_nats": 1.552840232849121, "step": 1010, "unclipped_grad_norm": 1.048907995223999 }, { "bits_per_byte": 2.2041726491204714, "learning_rate": 0.00029559058224103895, "loss_nats": 1.5278160572052002, "step": 1020, "unclipped_grad_norm": 1.0486118793487549 }, { "bits_per_byte": 2.216881995036582, "learning_rate": 0.0002954480070625017, "loss_nats": 1.5366255044937134, "step": 1030, "unclipped_grad_norm": 0.9243784546852112 }, { "bits_per_byte": 2.2610579746851553, "learning_rate": 0.00029530320288744205, "loss_nats": 1.5672459602355957, "step": 1040, "unclipped_grad_norm": 1.102870225906372 }, { "bits_per_byte": 2.2528270569975124, "learning_rate": 0.0002951561721901638, "loss_nats": 1.5615407228469849, "step": 1050, "unclipped_grad_norm": 1.0764544010162354 }, { "bits_per_byte": 2.14889054582713, "learning_rate": 0.00029500691748301594, "loss_nats": 1.489497423171997, "step": 1060, "unclipped_grad_norm": 0.971049427986145 }, { "bits_per_byte": 2.2350196293618754, "learning_rate": 0.00029485544131634963, "loss_nats": 1.5491975545883179, "step": 1070, "unclipped_grad_norm": 1.0203380584716797 }, { "bits_per_byte": 2.217042282867186, "learning_rate": 0.0002947017462784745, "loss_nats": 1.5367366075515747, "step": 1080, "unclipped_grad_norm": 1.023221492767334 }, { "bits_per_byte": 2.186512266651322, "learning_rate": 0.00029454583499561476, "loss_nats": 1.5155748128890991, "step": 1090, "unclipped_grad_norm": 1.111006498336792 }, { "bits_per_byte": 2.2323391077655277, "learning_rate": 0.00029438771013186413, "loss_nats": 1.5473395586013794, "step": 1100, "unclipped_grad_norm": 0.9547863602638245 }, { "bits_per_byte": 2.116467344590568, "learning_rate": 0.0002942273743891401, "loss_nats": 1.4670233726501465, "step": 1110, "unclipped_grad_norm": 1.1085364818572998 }, { "bits_per_byte": 2.143238164023873, "learning_rate": 0.00029406483050713824, "loss_nats": 1.485579490661621, "step": 1120, "unclipped_grad_norm": 0.901618242263794 }, { "bits_per_byte": 2.077939275196329, "learning_rate": 0.00029390008126328504, "loss_nats": 1.4403177499771118, "step": 1130, "unclipped_grad_norm": 0.8683149814605713 }, { "bits_per_byte": 2.224320932617002, "learning_rate": 0.00029373312947269045, "loss_nats": 1.5417817831039429, "step": 1140, "unclipped_grad_norm": 1.030465006828308 }, { "bits_per_byte": 2.0642131678483175, "learning_rate": 0.00029356397798809995, "loss_nats": 1.4308035373687744, "step": 1150, "unclipped_grad_norm": 0.827274739742279 }, { "bits_per_byte": 2.103388407957753, "learning_rate": 0.0002933926296998457, "loss_nats": 1.4579577445983887, "step": 1160, "unclipped_grad_norm": 0.9158841967582703 }, { "bits_per_byte": 2.1566975262803965, "learning_rate": 0.0002932190875357972, "loss_nats": 1.4949088096618652, "step": 1170, "unclipped_grad_norm": 0.9397322535514832 }, { "bits_per_byte": 2.1322636071070153, "learning_rate": 0.00029304335446131126, "loss_nats": 1.4779725074768066, "step": 1180, "unclipped_grad_norm": 0.8510220050811768 }, { "bits_per_byte": 2.068760561119781, "learning_rate": 0.00029286543347918116, "loss_nats": 1.4339555501937866, "step": 1190, "unclipped_grad_norm": 0.9270830154418945 }, { "bits_per_byte": 2.0310528489701576, "learning_rate": 0.0002926853276295856, "loss_nats": 1.4078185558319092, "step": 1200, "unclipped_grad_norm": 0.8815417885780334 }, { "bits_per_byte": 2.1008361854189492, "learning_rate": 0.0002925030399900368, "loss_nats": 1.456188678741455, "step": 1210, "unclipped_grad_norm": 1.2037686109542847 }, { "bits_per_byte": 2.1123913557651197, "learning_rate": 0.00029231857367532726, "loss_nats": 1.464198112487793, "step": 1220, "unclipped_grad_norm": 0.8964646458625793 }, { "bits_per_byte": 2.0711254945517954, "learning_rate": 0.0002921319318374776, "loss_nats": 1.4355947971343994, "step": 1230, "unclipped_grad_norm": 0.8661414980888367 }, { "bits_per_byte": 2.0159510523978943, "learning_rate": 0.00029194311766568174, "loss_nats": 1.397350788116455, "step": 1240, "unclipped_grad_norm": 0.9392534494400024 }, { "bits_per_byte": 2.104896867788471, "learning_rate": 0.000291752134386253, "loss_nats": 1.4590033292770386, "step": 1250, "unclipped_grad_norm": 0.8430818915367126 }, { "bits_per_byte": 2.08180441329182, "learning_rate": 0.00029155898526256856, "loss_nats": 1.442996859550476, "step": 1260, "unclipped_grad_norm": 0.8917548656463623 }, { "bits_per_byte": 2.087352401626003, "learning_rate": 0.0002913636735950141, "loss_nats": 1.4468424320220947, "step": 1270, "unclipped_grad_norm": 0.9866762161254883 }, { "bits_per_byte": 1.9607164163161914, "learning_rate": 0.0002911662027209272, "loss_nats": 1.359065055847168, "step": 1280, "unclipped_grad_norm": 0.8596939444541931 }, { "bits_per_byte": 2.088993460080536, "learning_rate": 0.00029096657601454013, "loss_nats": 1.4479799270629883, "step": 1290, "unclipped_grad_norm": 1.007126808166504 }, { "bits_per_byte": 2.034349412421896, "learning_rate": 0.00029076479688692266, "loss_nats": 1.4101035594940186, "step": 1300, "unclipped_grad_norm": 0.8474438786506653 }, { "bits_per_byte": 2.0247273270714237, "learning_rate": 0.00029056086878592323, "loss_nats": 1.4034340381622314, "step": 1310, "unclipped_grad_norm": 0.8226730823516846 }, { "bits_per_byte": 2.020511000402871, "learning_rate": 0.00029035479519611045, "loss_nats": 1.4005115032196045, "step": 1320, "unclipped_grad_norm": 0.9414357542991638 }, { "bits_per_byte": 2.085571349293668, "learning_rate": 0.0002901465796387134, "loss_nats": 1.4456079006195068, "step": 1330, "unclipped_grad_norm": 0.9917200207710266 }, { "bits_per_byte": 2.0422444479924042, "learning_rate": 0.00028993622567156154, "loss_nats": 1.4155759811401367, "step": 1340, "unclipped_grad_norm": 0.8823419213294983 }, { "bits_per_byte": 2.0447034559344406, "learning_rate": 0.0002897237368890237, "loss_nats": 1.4172804355621338, "step": 1350, "unclipped_grad_norm": 0.8861486315727234 }, { "bits_per_byte": 2.0988803987133373, "learning_rate": 0.0002895091169219469, "loss_nats": 1.4548330307006836, "step": 1360, "unclipped_grad_norm": 0.9090827107429504 }, { "bits_per_byte": 2.082760980795919, "learning_rate": 0.00028929236943759425, "loss_nats": 1.4436599016189575, "step": 1370, "unclipped_grad_norm": 0.8366561532020569 }, { "bits_per_byte": 2.0317923743688677, "learning_rate": 0.00028907349813958225, "loss_nats": 1.4083311557769775, "step": 1380, "unclipped_grad_norm": 0.8379060626029968 }, { "bits_per_byte": 2.0432778917414387, "learning_rate": 0.0002888525067678175, "loss_nats": 1.4162923097610474, "step": 1390, "unclipped_grad_norm": 0.8665340542793274 }, { "bits_per_byte": 1.9746810636009497, "learning_rate": 0.00028862939909843273, "loss_nats": 1.3687446117401123, "step": 1400, "unclipped_grad_norm": 0.8112426996231079 }, { "bits_per_byte": 2.0034977886489185, "learning_rate": 0.0002884041789437225, "loss_nats": 1.388718843460083, "step": 1410, "unclipped_grad_norm": 0.8745980262756348 }, { "bits_per_byte": 2.019574210903621, "learning_rate": 0.0002881768501520778, "loss_nats": 1.3998621702194214, "step": 1420, "unclipped_grad_norm": 0.8596643209457397 }, { "bits_per_byte": 2.0775151659793014, "learning_rate": 0.0002879474166079205, "loss_nats": 1.4400237798690796, "step": 1430, "unclipped_grad_norm": 0.913244903087616 }, { "bits_per_byte": 2.027370700415185, "learning_rate": 0.00028771588223163694, "loss_nats": 1.405266284942627, "step": 1440, "unclipped_grad_norm": 1.0067864656448364 }, { "bits_per_byte": 1.9631205617926026, "learning_rate": 0.00028748225097951054, "loss_nats": 1.3607314825057983, "step": 1450, "unclipped_grad_norm": 0.8125852942466736 }, { "bits_per_byte": 1.9959419028659118, "learning_rate": 0.000287246526843655, "loss_nats": 1.383481502532959, "step": 1460, "unclipped_grad_norm": 0.9857028126716614 }, { "bits_per_byte": 1.9364156117145783, "learning_rate": 0.00028700871385194533, "loss_nats": 1.3422210216522217, "step": 1470, "unclipped_grad_norm": 0.8698804974555969 }, { "bits_per_byte": 1.996789089404062, "learning_rate": 0.00028676881606794945, "loss_nats": 1.3840687274932861, "step": 1480, "unclipped_grad_norm": 0.8397032022476196 }, { "bits_per_byte": 2.017818268038312, "learning_rate": 0.00028652683759085854, "loss_nats": 1.398645043373108, "step": 1490, "unclipped_grad_norm": 0.8181726932525635 }, { "bits_per_byte": 1.996446843928845, "learning_rate": 0.0002862827825554171, "loss_nats": 1.38383150100708, "step": 1500, "unclipped_grad_norm": 0.8242432475090027 }, { "bits_per_byte": 2.0162594172908914, "learning_rate": 0.00028603665513185245, "loss_nats": 1.3975645303726196, "step": 1510, "unclipped_grad_norm": 0.9299814701080322 }, { "bits_per_byte": 1.980743108078562, "learning_rate": 0.000285788459525803, "loss_nats": 1.3729465007781982, "step": 1520, "unclipped_grad_norm": 0.8446369171142578 }, { "bits_per_byte": 2.047233664693934, "learning_rate": 0.00028553819997824707, "loss_nats": 1.4190342426300049, "step": 1530, "unclipped_grad_norm": 0.9717140197753906 }, { "bits_per_byte": 1.993557363411699, "learning_rate": 0.00028528588076542964, "loss_nats": 1.3818286657333374, "step": 1540, "unclipped_grad_norm": 0.8417568802833557 }, { "bits_per_byte": 2.049996737962696, "learning_rate": 0.00028503150619878996, "loss_nats": 1.4209494590759277, "step": 1550, "unclipped_grad_norm": 0.8546327352523804 }, { "bits_per_byte": 1.966114779744124, "learning_rate": 0.0002847750806248876, "loss_nats": 1.3628069162368774, "step": 1560, "unclipped_grad_norm": 0.9258830547332764 }, { "bits_per_byte": 2.0427641795633114, "learning_rate": 0.000284516608425328, "loss_nats": 1.4159362316131592, "step": 1570, "unclipped_grad_norm": 0.8567822575569153 }, { "bits_per_byte": 1.9830621221427962, "learning_rate": 0.000284256094016688, "loss_nats": 1.374553918838501, "step": 1580, "unclipped_grad_norm": 0.8522606492042542 }, { "bits_per_byte": 1.9876789964052082, "learning_rate": 0.0002839935418504399, "loss_nats": 1.3777540922164917, "step": 1590, "unclipped_grad_norm": 0.8731296062469482 }, { "bits_per_byte": 2.0122188568915207, "learning_rate": 0.000283728956412876, "loss_nats": 1.3947638273239136, "step": 1600, "unclipped_grad_norm": 0.8627029657363892 }, { "bits_per_byte": 1.9498011934138815, "learning_rate": 0.00028346234222503136, "loss_nats": 1.3514991998672485, "step": 1610, "unclipped_grad_norm": 0.7740172147750854 }, { "bits_per_byte": 1.9095543294069197, "learning_rate": 0.00028319370384260675, "loss_nats": 1.3236021995544434, "step": 1620, "unclipped_grad_norm": 0.7910023331642151 }, { "bits_per_byte": 1.9407452749500496, "learning_rate": 0.00028292304585589105, "loss_nats": 1.3452221155166626, "step": 1630, "unclipped_grad_norm": 0.8484107851982117 }, { "bits_per_byte": 1.9911346438089947, "learning_rate": 0.0002826503728896824, "loss_nats": 1.3801493644714355, "step": 1640, "unclipped_grad_norm": 0.8722788691520691 }, { "bits_per_byte": 2.0368601871418424, "learning_rate": 0.0002823756896032095, "loss_nats": 1.4118438959121704, "step": 1650, "unclipped_grad_norm": 0.8482462763786316 }, { "bits_per_byte": 1.9665676100638458, "learning_rate": 0.0002820990006900518, "loss_nats": 1.3631207942962646, "step": 1660, "unclipped_grad_norm": 0.8444085121154785 }, { "bits_per_byte": 1.9945896032821775, "learning_rate": 0.00028182031087805947, "loss_nats": 1.3825441598892212, "step": 1670, "unclipped_grad_norm": 0.8560684323310852 }, { "bits_per_byte": 2.0061820938635853, "learning_rate": 0.0002815396249292725, "loss_nats": 1.3905794620513916, "step": 1680, "unclipped_grad_norm": 0.8526912927627563 }, { "bits_per_byte": 2.0514603103215387, "learning_rate": 0.0002812569476398391, "loss_nats": 1.4219639301300049, "step": 1690, "unclipped_grad_norm": 0.8917502760887146 }, { "bits_per_byte": 1.9422809080396035, "learning_rate": 0.00028097228383993446, "loss_nats": 1.3462865352630615, "step": 1700, "unclipped_grad_norm": 0.7474871873855591 }, { "bits_per_byte": 1.878015978909047, "learning_rate": 0.0002806856383936774, "loss_nats": 1.3017414808273315, "step": 1710, "unclipped_grad_norm": 0.8072315454483032 }, { "bits_per_byte": 1.8905051870320533, "learning_rate": 0.00028039701619904774, "loss_nats": 1.3103983402252197, "step": 1720, "unclipped_grad_norm": 0.845556914806366 }, { "bits_per_byte": 1.8762399140735866, "learning_rate": 0.0002801064221878024, "loss_nats": 1.3005104064941406, "step": 1730, "unclipped_grad_norm": 0.9344940185546875 }, { "bits_per_byte": 1.9632232354351677, "learning_rate": 0.00027981386132539134, "loss_nats": 1.3608026504516602, "step": 1740, "unclipped_grad_norm": 0.7860697507858276 }, { "bits_per_byte": 1.989613629244764, "learning_rate": 0.0002795193386108724, "loss_nats": 1.3790950775146484, "step": 1750, "unclipped_grad_norm": 0.8348115086555481 }, { "bits_per_byte": 1.9869124697303127, "learning_rate": 0.00027922285907682625, "loss_nats": 1.3772227764129639, "step": 1760, "unclipped_grad_norm": 0.8862095475196838 }, { "bits_per_byte": 1.9928130224987648, "learning_rate": 0.00027892442778927005, "loss_nats": 1.3813127279281616, "step": 1770, "unclipped_grad_norm": 0.816246509552002 }, { "bits_per_byte": 1.8514405317671223, "learning_rate": 0.00027862404984757116, "loss_nats": 1.2833207845687866, "step": 1780, "unclipped_grad_norm": 0.8118871450424194 }, { "bits_per_byte": 1.9588221993995885, "learning_rate": 0.0002783217303843597, "loss_nats": 1.3577520847320557, "step": 1790, "unclipped_grad_norm": 0.839783251285553 }, { "bits_per_byte": 1.8827697513615458, "learning_rate": 0.00027801747456544134, "loss_nats": 1.3050365447998047, "step": 1800, "unclipped_grad_norm": 0.8949951529502869 }, { "bits_per_byte": 1.92937567388416, "learning_rate": 0.00027771128758970843, "loss_nats": 1.33734130859375, "step": 1810, "unclipped_grad_norm": 0.7295374274253845 }, { "bits_per_byte": 1.8974312723476003, "learning_rate": 0.00027740317468905165, "loss_nats": 1.3151991367340088, "step": 1820, "unclipped_grad_norm": 0.7755441069602966 }, { "bits_per_byte": 1.8886371114783813, "learning_rate": 0.0002770931411282704, "loss_nats": 1.3091034889221191, "step": 1830, "unclipped_grad_norm": 0.7666956186294556 }, { "bits_per_byte": 1.8876932706904463, "learning_rate": 0.0002767811922049827, "loss_nats": 1.3084492683410645, "step": 1840, "unclipped_grad_norm": 0.8784642219543457 }, { "bits_per_byte": 1.8964055678178544, "learning_rate": 0.00027646733324953503, "loss_nats": 1.314488172531128, "step": 1850, "unclipped_grad_norm": 0.8603965044021606 }, { "bits_per_byte": 1.8927124123732264, "learning_rate": 0.00027615156962491097, "loss_nats": 1.3119282722473145, "step": 1860, "unclipped_grad_norm": 0.8056252002716064 }, { "bits_per_byte": 1.8818489562488263, "learning_rate": 0.0002758339067266396, "loss_nats": 1.3043982982635498, "step": 1870, "unclipped_grad_norm": 0.8151036500930786 }, { "bits_per_byte": 1.8811881989042114, "learning_rate": 0.0002755143499827035, "loss_nats": 1.3039402961730957, "step": 1880, "unclipped_grad_norm": 0.79389888048172 }, { "bits_per_byte": 1.8949089747900463, "learning_rate": 0.0002751929048534456, "loss_nats": 1.313450813293457, "step": 1890, "unclipped_grad_norm": 0.8192400932312012 }, { "bits_per_byte": 1.9233574849825177, "learning_rate": 0.0002748695768314763, "loss_nats": 1.3331698179244995, "step": 1900, "unclipped_grad_norm": 0.7950363159179688 }, { "bits_per_byte": 1.8611783614416253, "learning_rate": 0.00027454437144157947, "loss_nats": 1.2900705337524414, "step": 1910, "unclipped_grad_norm": 0.8037525415420532 }, { "bits_per_byte": 1.855479888287937, "learning_rate": 0.00027421729424061787, "loss_nats": 1.2861206531524658, "step": 1920, "unclipped_grad_norm": 0.7507341504096985 }, { "bits_per_byte": 1.9602173226633874, "learning_rate": 0.0002738883508174385, "loss_nats": 1.3587191104888916, "step": 1930, "unclipped_grad_norm": 0.8329893350601196 }, { "bits_per_byte": 1.8066889261861108, "learning_rate": 0.0002735575467927768, "loss_nats": 1.2523013353347778, "step": 1940, "unclipped_grad_norm": 0.7595474123954773 }, { "bits_per_byte": 1.9033319970987033, "learning_rate": 0.0002732248878191609, "loss_nats": 1.319289207458496, "step": 1950, "unclipped_grad_norm": 0.7690061330795288 }, { "bits_per_byte": 1.816397862775269, "learning_rate": 0.0002728903795808148, "loss_nats": 1.259031057357788, "step": 1960, "unclipped_grad_norm": 0.7758741974830627 }, { "bits_per_byte": 1.909067618504978, "learning_rate": 0.0002725540277935614, "loss_nats": 1.3232648372650146, "step": 1970, "unclipped_grad_norm": 0.7503845691680908 }, { "bits_per_byte": 1.9405849871194454, "learning_rate": 0.00027221583820472456, "loss_nats": 1.3451110124588013, "step": 1980, "unclipped_grad_norm": 0.8655099868774414 }, { "bits_per_byte": 1.9436471382180576, "learning_rate": 0.0002718758165930314, "loss_nats": 1.347233533859253, "step": 1990, "unclipped_grad_norm": 0.8436980247497559 }, { "bits_per_byte": 1.9015801818170146, "learning_rate": 0.0002715339687685131, "loss_nats": 1.3180749416351318, "step": 2000, "unclipped_grad_norm": 0.8021923303604126 }, { "bits_per_byte": 1.8746163978494417, "learning_rate": 0.00027119030057240576, "loss_nats": 1.2993850708007812, "step": 2010, "unclipped_grad_norm": 0.7083144187927246 }, { "bits_per_byte": 1.9072580170525995, "learning_rate": 0.0002708448178770506, "loss_nats": 1.3220105171203613, "step": 2020, "unclipped_grad_norm": 0.7678324580192566 }, { "bits_per_byte": 1.983610230851096, "learning_rate": 0.00027049752658579376, "loss_nats": 1.3749338388442993, "step": 2030, "unclipped_grad_norm": 0.8317387104034424 }, { "bits_per_byte": 1.909348466173837, "learning_rate": 0.00027014843263288514, "loss_nats": 1.323459506034851, "step": 2040, "unclipped_grad_norm": 0.8302199840545654 }, { "bits_per_byte": 1.961244747019642, "learning_rate": 0.00026979754198337724, "loss_nats": 1.359431266784668, "step": 2050, "unclipped_grad_norm": 0.7990598082542419 }, { "bits_per_byte": 1.9220292629699043, "learning_rate": 0.00026944486063302305, "loss_nats": 1.3322491645812988, "step": 2060, "unclipped_grad_norm": 0.8404503464698792 }, { "bits_per_byte": 1.8663830724046866, "learning_rate": 0.0002690903946081738, "loss_nats": 1.2936781644821167, "step": 2070, "unclipped_grad_norm": 0.7720745801925659 }, { "bits_per_byte": 1.8672120287818452, "learning_rate": 0.00026873414996567596, "loss_nats": 1.2942527532577515, "step": 2080, "unclipped_grad_norm": 0.8028935790061951 }, { "bits_per_byte": 1.857879218250124, "learning_rate": 0.00026837613279276736, "loss_nats": 1.2877837419509888, "step": 2090, "unclipped_grad_norm": 0.7713416814804077 }, { "bits_per_byte": 1.8603779541845098, "learning_rate": 0.0002680163492069739, "loss_nats": 1.289515733718872, "step": 2100, "unclipped_grad_norm": 0.7443708181381226 }, { "bits_per_byte": 1.897931741861611, "learning_rate": 0.00026765480535600414, "loss_nats": 1.3155460357666016, "step": 2110, "unclipped_grad_norm": 0.7534405589103699 }, { "bits_per_byte": 1.8710701155886507, "learning_rate": 0.0002672915074176452, "loss_nats": 1.2969269752502441, "step": 2120, "unclipped_grad_norm": 0.7505779266357422 }, { "bits_per_byte": 1.8349618420920515, "learning_rate": 0.00026692646159965624, "loss_nats": 1.271898627281189, "step": 2130, "unclipped_grad_norm": 0.7835680246353149 }, { "bits_per_byte": 1.9616896661374241, "learning_rate": 0.0002665596741396633, "loss_nats": 1.3597396612167358, "step": 2140, "unclipped_grad_norm": 0.7786996960639954 }, { "bits_per_byte": 1.8168395142226847, "learning_rate": 0.000266191151305052, "loss_nats": 1.2593371868133545, "step": 2150, "unclipped_grad_norm": 0.7194197773933411 }, { "bits_per_byte": 1.8739360344826286, "learning_rate": 0.00026582089939286075, "loss_nats": 1.2989134788513184, "step": 2160, "unclipped_grad_norm": 0.8012002110481262 }, { "bits_per_byte": 1.9634492206384013, "learning_rate": 0.0002654489247296732, "loss_nats": 1.3609592914581299, "step": 2170, "unclipped_grad_norm": 0.8270084261894226 }, { "bits_per_byte": 1.8175344961148212, "learning_rate": 0.00026507523367151007, "loss_nats": 1.2598189115524292, "step": 2180, "unclipped_grad_norm": 0.7824540734291077 }, { "bits_per_byte": 1.8458584908680682, "learning_rate": 0.00026469983260372037, "loss_nats": 1.279451608657837, "step": 2190, "unclipped_grad_norm": 0.7973959445953369 }, { "bits_per_byte": 1.8557901449900935, "learning_rate": 0.0002643227279408727, "loss_nats": 1.2863357067108154, "step": 2200, "unclipped_grad_norm": 0.7451788783073425 }, { "bits_per_byte": 1.8380872828061812, "learning_rate": 0.0002639439261266453, "loss_nats": 1.2740650177001953, "step": 2210, "unclipped_grad_norm": 0.7804975509643555 }, { "bits_per_byte": 1.8298283319464475, "learning_rate": 0.000263563433633716, "loss_nats": 1.2683403491973877, "step": 2220, "unclipped_grad_norm": 0.7694668173789978 }, { "bits_per_byte": 1.9031531351418058, "learning_rate": 0.00026318125696365156, "loss_nats": 1.3191652297973633, "step": 2230, "unclipped_grad_norm": 0.8418469429016113 }, { "bits_per_byte": 1.8400618156207371, "learning_rate": 0.000262797402646797, "loss_nats": 1.2754336595535278, "step": 2240, "unclipped_grad_norm": 0.7987614870071411 }, { "bits_per_byte": 1.852736936989326, "learning_rate": 0.00026241187724216334, "loss_nats": 1.2842193841934204, "step": 2250, "unclipped_grad_norm": 0.7795421481132507 }, { "bits_per_byte": 1.8339105121473271, "learning_rate": 0.0002620246873373162, "loss_nats": 1.271169900894165, "step": 2260, "unclipped_grad_norm": 0.7681040167808533 }, { "bits_per_byte": 1.825916414569922, "learning_rate": 0.00026163583954826264, "loss_nats": 1.2656288146972656, "step": 2270, "unclipped_grad_norm": 0.7933830618858337 }, { "bits_per_byte": 1.8451832869807807, "learning_rate": 0.0002612453405193386, "loss_nats": 1.2789835929870605, "step": 2280, "unclipped_grad_norm": 0.7935850024223328 }, { "bits_per_byte": 1.8711039961708706, "learning_rate": 0.000260853196923095, "loss_nats": 1.2969504594802856, "step": 2290, "unclipped_grad_norm": 0.7818295955657959 }, { "bits_per_byte": 1.7944778140095496, "learning_rate": 0.0002604594154601839, "loss_nats": 1.2438372373580933, "step": 2300, "unclipped_grad_norm": 0.7245032787322998 }, { "bits_per_byte": 1.8221434591752936, "learning_rate": 0.0002600640028592441, "loss_nats": 1.2630136013031006, "step": 2310, "unclipped_grad_norm": 0.7396021485328674 }, { "bits_per_byte": 1.7928696042413317, "learning_rate": 0.0002596669658767859, "loss_nats": 1.242722511291504, "step": 2320, "unclipped_grad_norm": 0.7618010640144348 }, { "bits_per_byte": 1.8702191454321815, "learning_rate": 0.000259268311297076, "loss_nats": 1.2963371276855469, "step": 2330, "unclipped_grad_norm": 0.7263991236686707 }, { "bits_per_byte": 1.7994940319872652, "learning_rate": 0.00025886804593202114, "loss_nats": 1.247314214706421, "step": 2340, "unclipped_grad_norm": 0.7788637280464172 }, { "bits_per_byte": 1.849551646312696, "learning_rate": 0.00025846617662105193, "loss_nats": 1.2820115089416504, "step": 2350, "unclipped_grad_norm": 0.7732409238815308 }, { "bits_per_byte": 1.8613248906601605, "learning_rate": 0.00025806271023100637, "loss_nats": 1.2901721000671387, "step": 2360, "unclipped_grad_norm": 0.8177725672721863 }, { "bits_per_byte": 1.8822589628884834, "learning_rate": 0.00025765765365601165, "loss_nats": 1.3046824932098389, "step": 2370, "unclipped_grad_norm": 0.7931574583053589 }, { "bits_per_byte": 1.8382422391746087, "learning_rate": 0.00025725101381736707, "loss_nats": 1.2741724252700806, "step": 2380, "unclipped_grad_norm": 0.8604201674461365 }, { "bits_per_byte": 1.8797817247854554, "learning_rate": 0.00025684279766342567, "loss_nats": 1.3029654026031494, "step": 2390, "unclipped_grad_norm": 0.7636454105377197 }, { "bits_per_byte": 1.817860919186159, "learning_rate": 0.00025643301216947504, "loss_nats": 1.2600451707839966, "step": 2400, "unclipped_grad_norm": 0.779283881187439 }, { "bits_per_byte": 1.8354538844561699, "learning_rate": 0.00025602166433761866, "loss_nats": 1.2722396850585938, "step": 2410, "unclipped_grad_norm": 0.7781111001968384 }, { "bits_per_byte": 1.8069649583407457, "learning_rate": 0.0002556087611966561, "loss_nats": 1.2524926662445068, "step": 2420, "unclipped_grad_norm": 0.7548633813858032 }, { "bits_per_byte": 1.8169932667125561, "learning_rate": 0.00025519430980196277, "loss_nats": 1.259443759918213, "step": 2430, "unclipped_grad_norm": 0.7562755942344666 }, { "bits_per_byte": 1.8760644917697067, "learning_rate": 0.0002547783172353695, "loss_nats": 1.3003888130187988, "step": 2440, "unclipped_grad_norm": 0.7478408813476562 }, { "bits_per_byte": 1.838183937055966, "learning_rate": 0.0002543607906050415, "loss_nats": 1.2741320133209229, "step": 2450, "unclipped_grad_norm": 0.760445237159729 }, { "bits_per_byte": 1.8195946762895068, "learning_rate": 0.0002539417370453568, "loss_nats": 1.261246919631958, "step": 2460, "unclipped_grad_norm": 0.7331485748291016 }, { "bits_per_byte": 1.8163858239897086, "learning_rate": 0.00025352116371678464, "loss_nats": 1.2590227127075195, "step": 2470, "unclipped_grad_norm": 0.7682369351387024 }, { "bits_per_byte": 1.8266683227194942, "learning_rate": 0.0002530990778057627, "loss_nats": 1.2661499977111816, "step": 2480, "unclipped_grad_norm": 0.7132375240325928 }, { "bits_per_byte": 1.8042156436840528, "learning_rate": 0.00025267548652457475, "loss_nats": 1.250586986541748, "step": 2490, "unclipped_grad_norm": 0.7770085334777832 }, { "bits_per_byte": 1.8313916542427904, "learning_rate": 0.00025225039711122696, "loss_nats": 1.2694239616394043, "step": 2500, "unclipped_grad_norm": 0.7582318782806396 }, { "bits_per_byte": 1.7788244690759694, "learning_rate": 0.00025182381682932464, "loss_nats": 1.2329871654510498, "step": 2510, "unclipped_grad_norm": 0.8006963133811951 }, { "bits_per_byte": 1.8448090527325032, "learning_rate": 0.0002513957529679477, "loss_nats": 1.278724193572998, "step": 2520, "unclipped_grad_norm": 0.7802237272262573 }, { "bits_per_byte": 1.855150369528884, "learning_rate": 0.00025096621284152656, "loss_nats": 1.2858922481536865, "step": 2530, "unclipped_grad_norm": 0.7603861689567566 }, { "bits_per_byte": 1.7974587892969547, "learning_rate": 0.0002505352037897168, "loss_nats": 1.245903491973877, "step": 2540, "unclipped_grad_norm": 0.7829954624176025 }, { "bits_per_byte": 1.8048358131230642, "learning_rate": 0.0002501027331772739, "loss_nats": 1.2510168552398682, "step": 2550, "unclipped_grad_norm": 0.8207260966300964 }, { "bits_per_byte": 1.8410618947355044, "learning_rate": 0.00024966880839392746, "loss_nats": 1.2761268615722656, "step": 2560, "unclipped_grad_norm": 0.8064310550689697 }, { "bits_per_byte": 1.8072459779922554, "learning_rate": 0.0002492334368542548, "loss_nats": 1.2526874542236328, "step": 2570, "unclipped_grad_norm": 0.7300699353218079 }, { "bits_per_byte": 1.8352736466380657, "learning_rate": 0.00024879662599755416, "loss_nats": 1.2721147537231445, "step": 2580, "unclipped_grad_norm": 0.7476873993873596 }, { "bits_per_byte": 1.793617556789934, "learning_rate": 0.00024835838328771803, "loss_nats": 1.2432409524917603, "step": 2590, "unclipped_grad_norm": 0.7210070490837097 }, { "bits_per_byte": 1.797604974550188, "learning_rate": 0.0002479187162131051, "loss_nats": 1.2460048198699951, "step": 2600, "unclipped_grad_norm": 0.7853913903236389 }, { "bits_per_byte": 1.7385629865436842, "learning_rate": 0.00024747763228641264, "loss_nats": 1.2050800323486328, "step": 2610, "unclipped_grad_norm": 0.7092626094818115 }, { "bits_per_byte": 1.8079149905041119, "learning_rate": 0.00024703513904454795, "loss_nats": 1.2531511783599854, "step": 2620, "unclipped_grad_norm": 0.7522015571594238 }, { "bits_per_byte": 1.8198622812942493, "learning_rate": 0.00024659124404849956, "loss_nats": 1.261432409286499, "step": 2630, "unclipped_grad_norm": 0.7675716876983643 }, { "bits_per_byte": 1.680096624469618, "learning_rate": 0.00024614595488320833, "loss_nats": 1.164554238319397, "step": 2640, "unclipped_grad_norm": 0.6825466156005859 }, { "bits_per_byte": 1.7638260340768748, "learning_rate": 0.00024569927915743745, "loss_nats": 1.2225910425186157, "step": 2650, "unclipped_grad_norm": 0.7630164623260498 }, { "bits_per_byte": 1.8201589513669878, "learning_rate": 0.0002452512245036427, "loss_nats": 1.2616380453109741, "step": 2660, "unclipped_grad_norm": 0.7772946953773499 }, { "bits_per_byte": 1.811244230659514, "learning_rate": 0.00024480179857784177, "loss_nats": 1.2554588317871094, "step": 2670, "unclipped_grad_norm": 0.7582975625991821 }, { "bits_per_byte": 1.74257860945868, "learning_rate": 0.00024435100905948387, "loss_nats": 1.207863450050354, "step": 2680, "unclipped_grad_norm": 0.7818799614906311 }, { "bits_per_byte": 1.9030688636428832, "learning_rate": 0.00024389886365131793, "loss_nats": 1.3191068172454834, "step": 2690, "unclipped_grad_norm": 0.7075229287147522 }, { "bits_per_byte": 1.8200497423836899, "learning_rate": 0.00024344537007926163, "loss_nats": 1.2615623474121094, "step": 2700, "unclipped_grad_norm": 0.8272333741188049 }, { "bits_per_byte": 1.7354167359288, "learning_rate": 0.00024299053609226878, "loss_nats": 1.2028992176055908, "step": 2710, "unclipped_grad_norm": 0.7310962677001953 }, { "bits_per_byte": 1.728331394678554, "learning_rate": 0.00024253436946219733, "loss_nats": 1.1979880332946777, "step": 2720, "unclipped_grad_norm": 0.7193791270256042 }, { "bits_per_byte": 1.76859700479446, "learning_rate": 0.0002420768779836764, "loss_nats": 1.225898027420044, "step": 2730, "unclipped_grad_norm": 0.773657500743866 }, { "bits_per_byte": 1.7790022991369616, "learning_rate": 0.00024161806947397305, "loss_nats": 1.2331104278564453, "step": 2740, "unclipped_grad_norm": 0.7759174704551697 }, { "bits_per_byte": 1.7909771071512373, "learning_rate": 0.00024115795177285886, "loss_nats": 1.241410732269287, "step": 2750, "unclipped_grad_norm": 0.770866870880127 }, { "bits_per_byte": 1.81886375002334, "learning_rate": 0.0002406965327424758, "loss_nats": 1.2607402801513672, "step": 2760, "unclipped_grad_norm": 0.7793042659759521 }, { "bits_per_byte": 1.704546881994831, "learning_rate": 0.00024023382026720197, "loss_nats": 1.181501865386963, "step": 2770, "unclipped_grad_norm": 0.7179723978042603 }, { "bits_per_byte": 1.7222445846992174, "learning_rate": 0.00023976982225351698, "loss_nats": 1.1937689781188965, "step": 2780, "unclipped_grad_norm": 0.711125910282135 }, { "bits_per_byte": 1.8073385046584196, "learning_rate": 0.0002393045466298666, "loss_nats": 1.2527515888214111, "step": 2790, "unclipped_grad_norm": 0.7564381957054138 }, { "bits_per_byte": 1.8564130661515186, "learning_rate": 0.00023883800134652763, "loss_nats": 1.2867674827575684, "step": 2800, "unclipped_grad_norm": 0.7982874512672424 }, { "bits_per_byte": 1.7924368958917607, "learning_rate": 0.0002383701943754715, "loss_nats": 1.2424225807189941, "step": 2810, "unclipped_grad_norm": 0.827074408531189 }, { "bits_per_byte": 1.781587886310033, "learning_rate": 0.00023790113371022884, "loss_nats": 1.2349026203155518, "step": 2820, "unclipped_grad_norm": 0.6945767998695374 }, { "bits_per_byte": 1.862601517877515, "learning_rate": 0.0002374308273657521, "loss_nats": 1.2910569906234741, "step": 2830, "unclipped_grad_norm": 0.7473636269569397 }, { "bits_per_byte": 1.7520161574247766, "learning_rate": 0.00023695928337827911, "loss_nats": 1.2144050598144531, "step": 2840, "unclipped_grad_norm": 0.7269325852394104 }, { "bits_per_byte": 1.7726036846116109, "learning_rate": 0.00023648650980519564, "loss_nats": 1.2286752462387085, "step": 2850, "unclipped_grad_norm": 0.7537488341331482 }, { "bits_per_byte": 1.7375121725469125, "learning_rate": 0.00023601251472489757, "loss_nats": 1.2043516635894775, "step": 2860, "unclipped_grad_norm": 0.7839256525039673 }, { "bits_per_byte": 1.836773851301542, "learning_rate": 0.00023553730623665305, "loss_nats": 1.273154616355896, "step": 2870, "unclipped_grad_norm": 0.7615643739700317 }, { "bits_per_byte": 1.8235200083127996, "learning_rate": 0.00023506089246046403, "loss_nats": 1.263967752456665, "step": 2880, "unclipped_grad_norm": 0.7500733137130737 }, { "bits_per_byte": 1.7346435019305209, "learning_rate": 0.00023458328153692736, "loss_nats": 1.2023632526397705, "step": 2890, "unclipped_grad_norm": 0.7335168123245239 }, { "bits_per_byte": 1.710140445731493, "learning_rate": 0.0002341044816270961, "loss_nats": 1.1853790283203125, "step": 2900, "unclipped_grad_norm": 0.7237004041671753 }, { "bits_per_byte": 1.7640040361205178, "learning_rate": 0.00023362450091233953, "loss_nats": 1.2227144241333008, "step": 2910, "unclipped_grad_norm": 0.7653093338012695 }, { "bits_per_byte": 1.7372325287566095, "learning_rate": 0.00023314334759420388, "loss_nats": 1.204157829284668, "step": 2920, "unclipped_grad_norm": 0.7592934966087341 }, { "bits_per_byte": 1.7605243111456106, "learning_rate": 0.0002326610298942717, "loss_nats": 1.2203024625778198, "step": 2930, "unclipped_grad_norm": 0.7871323227882385 }, { "bits_per_byte": 1.7773966691085064, "learning_rate": 0.00023217755605402175, "loss_nats": 1.2319974899291992, "step": 2940, "unclipped_grad_norm": 0.7994735240936279 }, { "bits_per_byte": 1.761262116717812, "learning_rate": 0.00023169293433468802, "loss_nats": 1.2208138704299927, "step": 2950, "unclipped_grad_norm": 0.7672474384307861 }, { "bits_per_byte": 1.8009139207527887, "learning_rate": 0.00023120717301711857, "loss_nats": 1.2482984066009521, "step": 2960, "unclipped_grad_norm": 0.7721540927886963 }, { "bits_per_byte": 1.7577087831683356, "learning_rate": 0.00023072028040163408, "loss_nats": 1.218350887298584, "step": 2970, "unclipped_grad_norm": 0.7659915685653687 }, { "bits_per_byte": 1.7511909846659368, "learning_rate": 0.00023023226480788593, "loss_nats": 1.2138330936431885, "step": 2980, "unclipped_grad_norm": 0.7457980513572693 }, { "bits_per_byte": 1.7677185174138528, "learning_rate": 0.00022974313457471416, "loss_nats": 1.2252891063690186, "step": 2990, "unclipped_grad_norm": 0.7551211714744568 }, { "bits_per_byte": 1.7804380103063644, "learning_rate": 0.0002292528980600049, "loss_nats": 1.2341055870056152, "step": 3000, "unclipped_grad_norm": 0.7369211316108704 }, { "bits_per_byte": 1.77081558099059, "learning_rate": 0.00022876156364054758, "loss_nats": 1.227435827255249, "step": 3010, "unclipped_grad_norm": 0.7526711225509644 }, { "bits_per_byte": 1.726040929734363, "learning_rate": 0.0002282691397118918, "loss_nats": 1.1964004039764404, "step": 3020, "unclipped_grad_norm": 0.8113403916358948 }, { "bits_per_byte": 1.8045116258261877, "learning_rate": 0.00022777563468820376, "loss_nats": 1.250792145729065, "step": 3030, "unclipped_grad_norm": 0.7493385672569275 }, { "bits_per_byte": 1.7229166928987893, "learning_rate": 0.00022728105700212282, "loss_nats": 1.194234848022461, "step": 3040, "unclipped_grad_norm": 0.7606903314590454 }, { "bits_per_byte": 1.7403560776615798, "learning_rate": 0.00022678541510461698, "loss_nats": 1.2063229084014893, "step": 3050, "unclipped_grad_norm": 0.7690193057060242 }, { "bits_per_byte": 1.759874388708, "learning_rate": 0.00022628871746483876, "loss_nats": 1.2198519706726074, "step": 3060, "unclipped_grad_norm": 0.7480750679969788 }, { "bits_per_byte": 1.6873952242069339, "learning_rate": 0.0002257909725699804, "loss_nats": 1.169613242149353, "step": 3070, "unclipped_grad_norm": 0.7001752853393555 }, { "bits_per_byte": 1.737272084766308, "learning_rate": 0.00022529218892512886, "loss_nats": 1.2041852474212646, "step": 3080, "unclipped_grad_norm": 0.8322204351425171 }, { "bits_per_byte": 1.8015143121869506, "learning_rate": 0.00022479237505312034, "loss_nats": 1.248714566230774, "step": 3090, "unclipped_grad_norm": 0.8183287382125854 }, { "bits_per_byte": 1.7001176408045102, "learning_rate": 0.00022429153949439496, "loss_nats": 1.178431749343872, "step": 3100, "unclipped_grad_norm": 0.7750749588012695 }, { "bits_per_byte": 1.7770231227908324, "learning_rate": 0.00022378969080685053, "loss_nats": 1.231738567352295, "step": 3110, "unclipped_grad_norm": 0.7082787752151489 }, { "bits_per_byte": 1.749435729731231, "learning_rate": 0.0002232868375656965, "loss_nats": 1.2126164436340332, "step": 3120, "unclipped_grad_norm": 0.736611545085907 }, { "bits_per_byte": 1.7966217497352055, "learning_rate": 0.00022278298836330733, "loss_nats": 1.2453233003616333, "step": 3130, "unclipped_grad_norm": 0.7366204857826233 }, { "bits_per_byte": 1.7636881039908827, "learning_rate": 0.00022227815180907572, "loss_nats": 1.222495436668396, "step": 3140, "unclipped_grad_norm": 0.7583194375038147 }, { "bits_per_byte": 1.7359698321339747, "learning_rate": 0.00022177233652926546, "loss_nats": 1.2032825946807861, "step": 3150, "unclipped_grad_norm": 0.7600930333137512 }, { "bits_per_byte": 1.7415977924008093, "learning_rate": 0.00022126555116686416, "loss_nats": 1.207183599472046, "step": 3160, "unclipped_grad_norm": 0.7639689445495605 }, { "bits_per_byte": 1.654884311818432, "learning_rate": 0.00022075780438143532, "loss_nats": 1.1470783948898315, "step": 3170, "unclipped_grad_norm": 0.7011478543281555 }, { "bits_per_byte": 1.7584833930278216, "learning_rate": 0.00022024910484897065, "loss_nats": 1.2188878059387207, "step": 3180, "unclipped_grad_norm": 0.7645173072814941 }, { "bits_per_byte": 1.7305350083840592, "learning_rate": 0.00021973946126174158, "loss_nats": 1.199515461921692, "step": 3190, "unclipped_grad_norm": 0.7665988206863403 }, { "bits_per_byte": 1.7003714871971838, "learning_rate": 0.00021922888232815086, "loss_nats": 1.178607702255249, "step": 3200, "unclipped_grad_norm": 0.7379894852638245 }, { "bits_per_byte": 1.6959066455981344, "learning_rate": 0.0002187173767725837, "loss_nats": 1.1755129098892212, "step": 3210, "unclipped_grad_norm": 0.8013631701469421 }, { "bits_per_byte": 1.7098007799960389, "learning_rate": 0.0002182049533352588, "loss_nats": 1.1851435899734497, "step": 3220, "unclipped_grad_norm": 0.7296364903450012 }, { "bits_per_byte": 1.7987765203678656, "learning_rate": 0.00021769162077207884, "loss_nats": 1.246816873550415, "step": 3230, "unclipped_grad_norm": 0.7610434889793396 }, { "bits_per_byte": 1.6862874839527267, "learning_rate": 0.00021717738785448102, "loss_nats": 1.1688454151153564, "step": 3240, "unclipped_grad_norm": 0.723595917224884 }, { "bits_per_byte": 1.8345631863073515, "learning_rate": 0.00021666226336928708, "loss_nats": 1.2716223001480103, "step": 3250, "unclipped_grad_norm": 0.8353017568588257 }, { "bits_per_byte": 1.6872093109613513, "learning_rate": 0.0002161462561185532, "loss_nats": 1.1694843769073486, "step": 3260, "unclipped_grad_norm": 0.7602720260620117 }, { "bits_per_byte": 1.7198954736710823, "learning_rate": 0.00021562937491941957, "loss_nats": 1.1921406984329224, "step": 3270, "unclipped_grad_norm": 0.7516167163848877 }, { "bits_per_byte": 1.7692164863028679, "learning_rate": 0.00021511162860395975, "loss_nats": 1.2263274192810059, "step": 3280, "unclipped_grad_norm": 0.7487898468971252 }, { "bits_per_byte": 1.7624494849393688, "learning_rate": 0.0002145930260190298, "loss_nats": 1.2216368913650513, "step": 3290, "unclipped_grad_norm": 0.7708044052124023 }, { "bits_per_byte": 1.7002387165907176, "learning_rate": 0.00021407357602611698, "loss_nats": 1.1785156726837158, "step": 3300, "unclipped_grad_norm": 0.7703046202659607 }, { "bits_per_byte": 1.7332599015043295, "learning_rate": 0.0002135532875011885, "loss_nats": 1.2014042139053345, "step": 3310, "unclipped_grad_norm": 0.8132672905921936 }, { "bits_per_byte": 1.7262779218272521, "learning_rate": 0.0002130321693345397, "loss_nats": 1.1965646743774414, "step": 3320, "unclipped_grad_norm": 0.7545798420906067 }, { "bits_per_byte": 1.7272033604715449, "learning_rate": 0.00021251023043064224, "loss_nats": 1.1972061395645142, "step": 3330, "unclipped_grad_norm": 0.7775130867958069 }, { "bits_per_byte": 1.6975052243379043, "learning_rate": 0.00021198747970799188, "loss_nats": 1.1766209602355957, "step": 3340, "unclipped_grad_norm": 0.7105540633201599 }, { "bits_per_byte": 1.7083408192728646, "learning_rate": 0.00021146392609895618, "loss_nats": 1.1841316223144531, "step": 3350, "unclipped_grad_norm": 0.7163253426551819 }, { "bits_per_byte": 1.732425613664994, "learning_rate": 0.00021093957854962178, "loss_nats": 1.2008259296417236, "step": 3360, "unclipped_grad_norm": 0.7626740336418152 }, { "bits_per_byte": 1.6756070173688444, "learning_rate": 0.00021041444601964157, "loss_nats": 1.1614422798156738, "step": 3370, "unclipped_grad_norm": 0.8061453700065613 }, { "bits_per_byte": 1.7092863798873084, "learning_rate": 0.00020988853748208164, "loss_nats": 1.1847870349884033, "step": 3380, "unclipped_grad_norm": 0.7406735420227051 }, { "bits_per_byte": 1.7539413312185348, "learning_rate": 0.00020936186192326782, "loss_nats": 1.2157394886016846, "step": 3390, "unclipped_grad_norm": 0.7695116400718689 }, { "bits_per_byte": 1.8092177590843976, "learning_rate": 0.00020883442834263225, "loss_nats": 1.2540541887283325, "step": 3400, "unclipped_grad_norm": 0.7858016490936279 }, { "bits_per_byte": 1.8255297975707825, "learning_rate": 0.0002083062457525596, "loss_nats": 1.2653608322143555, "step": 3410, "unclipped_grad_norm": 0.7603964805603027 }, { "bits_per_byte": 1.6857877023693193, "learning_rate": 0.00020777732317823306, "loss_nats": 1.1684989929199219, "step": 3420, "unclipped_grad_norm": 0.7168062329292297 }, { "bits_per_byte": 1.6176447364444573, "learning_rate": 0.00020724766965748005, "loss_nats": 1.1212658882141113, "step": 3430, "unclipped_grad_norm": 0.7431134581565857 }, { "bits_per_byte": 1.711189711884407, "learning_rate": 0.00020671729424061788, "loss_nats": 1.1861063241958618, "step": 3440, "unclipped_grad_norm": 0.7753933668136597 }, { "bits_per_byte": 1.77535626693867, "learning_rate": 0.00020618620599029906, "loss_nats": 1.2305831909179688, "step": 3450, "unclipped_grad_norm": 0.8042635321617126 }, { "bits_per_byte": 1.7083845028661837, "learning_rate": 0.00020565441398135649, "loss_nats": 1.184161901473999, "step": 3460, "unclipped_grad_norm": 0.7639697194099426 }, { "bits_per_byte": 1.6937178224006033, "learning_rate": 0.00020512192730064827, "loss_nats": 1.1739957332611084, "step": 3470, "unclipped_grad_norm": 0.748127281665802 }, { "bits_per_byte": 1.6501064617948125, "learning_rate": 0.00020458875504690273, "loss_nats": 1.1437666416168213, "step": 3480, "unclipped_grad_norm": 0.7506517767906189 }, { "bits_per_byte": 1.8358463488654388, "learning_rate": 0.00020405490633056245, "loss_nats": 1.2725117206573486, "step": 3490, "unclipped_grad_norm": 0.836082398891449 }, { "bits_per_byte": 1.714104301868579, "learning_rate": 0.00020352039027362917, "loss_nats": 1.188126564025879, "step": 3500, "unclipped_grad_norm": 0.7544166445732117 }, { "bits_per_byte": 1.744907598516664, "learning_rate": 0.0002029852160095075, "loss_nats": 1.2094777822494507, "step": 3510, "unclipped_grad_norm": 0.7650617361068726 }, { "bits_per_byte": 1.738600994709525, "learning_rate": 0.000202449392682849, "loss_nats": 1.2051063776016235, "step": 3520, "unclipped_grad_norm": 0.7794367671012878 }, { "bits_per_byte": 1.7051804660806096, "learning_rate": 0.0002019129294493959, "loss_nats": 1.181941032409668, "step": 3530, "unclipped_grad_norm": 0.7445787787437439 }, { "bits_per_byte": 1.7158172490711725, "learning_rate": 0.00020137583547582478, "loss_nats": 1.1893138885498047, "step": 3540, "unclipped_grad_norm": 0.7629107236862183 }, { "bits_per_byte": 1.7173026632266746, "learning_rate": 0.00020083811993958973, "loss_nats": 1.1903434991836548, "step": 3550, "unclipped_grad_norm": 0.7763495445251465 }, { "bits_per_byte": 1.6560372835098163, "learning_rate": 0.00020029979202876563, "loss_nats": 1.14787757396698, "step": 3560, "unclipped_grad_norm": 0.7450886964797974 }, { "bits_per_byte": 1.6838996768281473, "learning_rate": 0.00019976086094189118, "loss_nats": 1.1671903133392334, "step": 3570, "unclipped_grad_norm": 0.7487124800682068 }, { "bits_per_byte": 1.6946102403759304, "learning_rate": 0.00019922133588781175, "loss_nats": 1.1746143102645874, "step": 3580, "unclipped_grad_norm": 0.757170557975769 }, { "bits_per_byte": 1.7307594457434352, "learning_rate": 0.0001986812260855219, "loss_nats": 1.1996710300445557, "step": 3590, "unclipped_grad_norm": 0.7700981497764587 }, { "bits_per_byte": 1.7491381997452384, "learning_rate": 0.00019814054076400787, "loss_nats": 1.2124102115631104, "step": 3600, "unclipped_grad_norm": 0.7877120971679688 }, { "bits_per_byte": 1.688025540622346, "learning_rate": 0.00019759928916209004, "loss_nats": 1.1700501441955566, "step": 3610, "unclipped_grad_norm": 0.780867874622345 }, { "bits_per_byte": 1.6436277033541595, "learning_rate": 0.00019705748052826493, "loss_nats": 1.1392759084701538, "step": 3620, "unclipped_grad_norm": 0.7423334717750549 }, { "bits_per_byte": 1.6950637586262558, "learning_rate": 0.00019651512412054723, "loss_nats": 1.1749286651611328, "step": 3630, "unclipped_grad_norm": 0.7513464689254761 }, { "bits_per_byte": 1.6758522646289749, "learning_rate": 0.00019597222920631144, "loss_nats": 1.1616122722625732, "step": 3640, "unclipped_grad_norm": 0.7453336715698242 }, { "bits_per_byte": 1.736063734661346, "learning_rate": 0.0001954288050621338, "loss_nats": 1.2033476829528809, "step": 3650, "unclipped_grad_norm": 0.7682862877845764 }, { "bits_per_byte": 1.6360068081291268, "learning_rate": 0.00019488486097363363, "loss_nats": 1.1339935064315796, "step": 3660, "unclipped_grad_norm": 0.7531437277793884 }, { "bits_per_byte": 1.7183529612754938, "learning_rate": 0.00019434040623531452, "loss_nats": 1.1910715103149414, "step": 3670, "unclipped_grad_norm": 0.7578480243682861 }, { "bits_per_byte": 1.6583936178092877, "learning_rate": 0.0001937954501504058, "loss_nats": 1.1495108604431152, "step": 3680, "unclipped_grad_norm": 0.7224898934364319 }, { "bits_per_byte": 1.6372521345040243, "learning_rate": 0.00019325000203070333, "loss_nats": 1.1348567008972168, "step": 3690, "unclipped_grad_norm": 0.7246464490890503 }, { "bits_per_byte": 1.6659130432703113, "learning_rate": 0.0001927040711964106, "loss_nats": 1.1547229290008545, "step": 3700, "unclipped_grad_norm": 0.7878638505935669 }, { "bits_per_byte": 1.694639821391879, "learning_rate": 0.00019215766697597923, "loss_nats": 1.1746348142623901, "step": 3710, "unclipped_grad_norm": 0.7455682754516602 }, { "bits_per_byte": 1.6891824679147003, "learning_rate": 0.00019161079870594978, "loss_nats": 1.1708520650863647, "step": 3720, "unclipped_grad_norm": 0.7515487670898438 }, { "bits_per_byte": 1.6091924770851538, "learning_rate": 0.0001910634757307922, "loss_nats": 1.1154072284698486, "step": 3730, "unclipped_grad_norm": 0.7353127598762512 }, { "bits_per_byte": 1.6633680440028435, "learning_rate": 0.0001905157074027459, "loss_nats": 1.152958869934082, "step": 3740, "unclipped_grad_norm": 0.7253232002258301 }, { "bits_per_byte": 1.6922951819126664, "learning_rate": 0.00018996750308166037, "loss_nats": 1.1730096340179443, "step": 3750, "unclipped_grad_norm": 0.7686370611190796 }, { "bits_per_byte": 1.7713081393026613, "learning_rate": 0.00018941887213483482, "loss_nats": 1.2277772426605225, "step": 3760, "unclipped_grad_norm": 0.7601476907730103 }, { "bits_per_byte": 1.7305630415561497, "learning_rate": 0.00018886982393685845, "loss_nats": 1.1995348930358887, "step": 3770, "unclipped_grad_norm": 0.7877833247184753 }, { "bits_per_byte": 1.5644979697442287, "learning_rate": 0.00018832036786945008, "loss_nats": 1.0844273567199707, "step": 3780, "unclipped_grad_norm": 0.7386898994445801 }, { "bits_per_byte": 1.6644766441703052, "learning_rate": 0.0001877705133212979, "loss_nats": 1.1537272930145264, "step": 3790, "unclipped_grad_norm": 0.7420179843902588 }, { "bits_per_byte": 1.6653939996300076, "learning_rate": 0.00018722026968789906, "loss_nats": 1.1543631553649902, "step": 3800, "unclipped_grad_norm": 0.7838358283042908 }, { "bits_per_byte": 1.6952983429620325, "learning_rate": 0.00018666964637139907, "loss_nats": 1.17509126663208, "step": 3810, "unclipped_grad_norm": 0.8048396110534668 }, { "bits_per_byte": 1.6744323758634516, "learning_rate": 0.00018611865278043115, "loss_nats": 1.160628080368042, "step": 3820, "unclipped_grad_norm": 0.7709245681762695 }, { "bits_per_byte": 1.720191799778519, "learning_rate": 0.00018556729832995555, "loss_nats": 1.1923460960388184, "step": 3830, "unclipped_grad_norm": 0.7672807574272156 }, { "bits_per_byte": 1.6467005173771254, "learning_rate": 0.00018501559244109864, "loss_nats": 1.1414058208465576, "step": 3840, "unclipped_grad_norm": 0.7319741249084473 }, { "bits_per_byte": 1.5964396195583748, "learning_rate": 0.00018446354454099178, "loss_nats": 1.106567621231079, "step": 3850, "unclipped_grad_norm": 0.7756319642066956 }, { "bits_per_byte": 1.6360811046342996, "learning_rate": 0.00018391116406261048, "loss_nats": 1.1340450048446655, "step": 3860, "unclipped_grad_norm": 0.766754150390625 }, { "bits_per_byte": 1.7433616464680584, "learning_rate": 0.00018335846044461305, "loss_nats": 1.2084062099456787, "step": 3870, "unclipped_grad_norm": 0.7916337847709656 }, { "bits_per_byte": 1.6358671582166262, "learning_rate": 0.0001828054431311793, "loss_nats": 1.1338967084884644, "step": 3880, "unclipped_grad_norm": 0.7497678399085999 }, { "bits_per_byte": 1.600699113872294, "learning_rate": 0.00018225212157184934, "loss_nats": 1.1095200777053833, "step": 3890, "unclipped_grad_norm": 0.7555440664291382 }, { "bits_per_byte": 1.6778517349279058, "learning_rate": 0.00018169850522136192, "loss_nats": 1.1629981994628906, "step": 3900, "unclipped_grad_norm": 0.826478123664856 }, { "bits_per_byte": 1.6805711246033486, "learning_rate": 0.000181144603539493, "loss_nats": 1.1648831367492676, "step": 3910, "unclipped_grad_norm": 0.7849931120872498 }, { "bits_per_byte": 1.6375147520118918, "learning_rate": 0.00018059042599089412, "loss_nats": 1.1350387334823608, "step": 3920, "unclipped_grad_norm": 0.7588120102882385 }, { "bits_per_byte": 1.673005091843941, "learning_rate": 0.00018003598204493048, "loss_nats": 1.15963876247406, "step": 3930, "unclipped_grad_norm": 0.742434024810791 }, { "bits_per_byte": 1.6996766772876981, "learning_rate": 0.00017948128117551936, "loss_nats": 1.1781260967254639, "step": 3940, "unclipped_grad_norm": 0.7588658928871155 }, { "bits_per_byte": 1.7339400928884918, "learning_rate": 0.00017892633286096816, "loss_nats": 1.2018756866455078, "step": 3950, "unclipped_grad_norm": 0.7937244176864624 }, { "bits_per_byte": 1.713467450112434, "learning_rate": 0.00017837114658381247, "loss_nats": 1.1876851320266724, "step": 3960, "unclipped_grad_norm": 0.7884036302566528 }, { "bits_per_byte": 1.6516962694193882, "learning_rate": 0.00017781573183065386, "loss_nats": 1.1448686122894287, "step": 3970, "unclipped_grad_norm": 0.8004661798477173 }, { "bits_per_byte": 1.6740500584305835, "learning_rate": 0.0001772600980919981, "loss_nats": 1.1603630781173706, "step": 3980, "unclipped_grad_norm": 0.7692103385925293 }, { "bits_per_byte": 1.691941929547794, "learning_rate": 0.0001767042548620927, "loss_nats": 1.172764778137207, "step": 3990, "unclipped_grad_norm": 0.7891059517860413 }, { "bits_per_byte": 1.5813677479673616, "learning_rate": 0.00017614821163876486, "loss_nats": 1.0961205959320068, "step": 4000, "unclipped_grad_norm": 0.7436221837997437 }, { "bits_per_byte": 1.6793110077204767, "learning_rate": 0.00017559197792325906, "loss_nats": 1.164009690284729, "step": 4010, "unclipped_grad_norm": 0.8044730424880981 }, { "bits_per_byte": 1.6584589712166156, "learning_rate": 0.0001750355632200748, "loss_nats": 1.1495561599731445, "step": 4020, "unclipped_grad_norm": 0.8183668851852417 }, { "bits_per_byte": 1.6850787898824628, "learning_rate": 0.00017447897703680418, "loss_nats": 1.1680076122283936, "step": 4030, "unclipped_grad_norm": 0.793128252029419 }, { "bits_per_byte": 1.617821878574846, "learning_rate": 0.0001739222288839694, "loss_nats": 1.1213886737823486, "step": 4040, "unclipped_grad_norm": 0.7582541108131409 }, { "bits_per_byte": 1.6450140555027646, "learning_rate": 0.00017336532827486026, "loss_nats": 1.1402368545532227, "step": 4050, "unclipped_grad_norm": 0.7559582591056824 }, { "bits_per_byte": 1.6581645369183384, "learning_rate": 0.00017280828472537163, "loss_nats": 1.1493520736694336, "step": 4060, "unclipped_grad_norm": 0.743245005607605 }, { "bits_per_byte": 1.680131880913045, "learning_rate": 0.0001722511077538408, "loss_nats": 1.1645786762237549, "step": 4070, "unclipped_grad_norm": 0.7624887824058533 }, { "bits_per_byte": 1.7564567494700543, "learning_rate": 0.00017169380688088497, "loss_nats": 1.2174830436706543, "step": 4080, "unclipped_grad_norm": 0.8132302761077881 }, { "bits_per_byte": 1.6669108866106175, "learning_rate": 0.0001711363916292384, "loss_nats": 1.1554145812988281, "step": 4090, "unclipped_grad_norm": 0.8276432752609253 }, { "bits_per_byte": 1.638365894151012, "learning_rate": 0.00017057887152358974, "loss_nats": 1.1356287002563477, "step": 4100, "unclipped_grad_norm": 0.7751885652542114 }, { "bits_per_byte": 1.6303674970073352, "learning_rate": 0.0001700212560904194, "loss_nats": 1.1300846338272095, "step": 4110, "unclipped_grad_norm": 0.7898787260055542 }, { "bits_per_byte": 1.6896222275529564, "learning_rate": 0.00016946355485783656, "loss_nats": 1.171156883239746, "step": 4120, "unclipped_grad_norm": 0.8148247599601746 }, { "bits_per_byte": 1.6213521664491068, "learning_rate": 0.0001689057773554166, "loss_nats": 1.1238356828689575, "step": 4130, "unclipped_grad_norm": 0.8133904933929443 }, { "bits_per_byte": 1.6881359534842002, "learning_rate": 0.00016834793311403806, "loss_nats": 1.1701266765594482, "step": 4140, "unclipped_grad_norm": 0.8217440247535706 }, { "bits_per_byte": 1.6129334437067215, "learning_rate": 0.00016779003166571981, "loss_nats": 1.1180002689361572, "step": 4150, "unclipped_grad_norm": 0.7947918176651001 }, { "bits_per_byte": 1.651558683298698, "learning_rate": 0.00016723208254345838, "loss_nats": 1.144773244857788, "step": 4160, "unclipped_grad_norm": 0.7860805988311768 }, { "bits_per_byte": 1.6828089628563754, "learning_rate": 0.00016667409528106481, "loss_nats": 1.1664342880249023, "step": 4170, "unclipped_grad_norm": 0.8207855224609375 }, { "bits_per_byte": 1.7311515661874024, "learning_rate": 0.00016611607941300184, "loss_nats": 1.1999428272247314, "step": 4180, "unclipped_grad_norm": 0.8373809456825256 }, { "bits_per_byte": 1.6433404923272186, "learning_rate": 0.0001655580444742211, "loss_nats": 1.139076828956604, "step": 4190, "unclipped_grad_norm": 0.7621071338653564 }, { "bits_per_byte": 1.682679287937625, "learning_rate": 0.00016499999999999997, "loss_nats": 1.166344404220581, "step": 4200, "unclipped_grad_norm": 0.7672834396362305 }, { "bits_per_byte": 1.661693964879345, "learning_rate": 0.0001644419555257789, "loss_nats": 1.1517984867095947, "step": 4210, "unclipped_grad_norm": 0.7330533862113953 }, { "bits_per_byte": 1.6972419188994332, "learning_rate": 0.0001638839205869981, "loss_nats": 1.1764384508132935, "step": 4220, "unclipped_grad_norm": 0.7866858243942261 }, { "bits_per_byte": 1.6670219874030747, "learning_rate": 0.0001633259047189352, "loss_nats": 1.155491590499878, "step": 4230, "unclipped_grad_norm": 0.7902593612670898 }, { "bits_per_byte": 1.6114956687455084, "learning_rate": 0.0001627679174565416, "loss_nats": 1.1170036792755127, "step": 4240, "unclipped_grad_norm": 0.740251362323761 }, { "bits_per_byte": 1.6335233786506678, "learning_rate": 0.00016220996833428016, "loss_nats": 1.1322721242904663, "step": 4250, "unclipped_grad_norm": 0.7777352333068848 }, { "bits_per_byte": 1.6115895712728796, "learning_rate": 0.00016165206688596191, "loss_nats": 1.1170687675476074, "step": 4260, "unclipped_grad_norm": 0.774424135684967 }, { "bits_per_byte": 1.619611874005026, "learning_rate": 0.00016109422264458335, "loss_nats": 1.1226294040679932, "step": 4270, "unclipped_grad_norm": 0.7803477048873901 }, { "bits_per_byte": 1.6299772683725275, "learning_rate": 0.00016053644514216336, "loss_nats": 1.1298141479492188, "step": 4280, "unclipped_grad_norm": 0.7580373287200928 }, { "bits_per_byte": 1.6795242662075467, "learning_rate": 0.00015997874390958058, "loss_nats": 1.164157509803772, "step": 4290, "unclipped_grad_norm": 0.7638103365898132 }, { "bits_per_byte": 1.600082040120998, "learning_rate": 0.00015942112847641018, "loss_nats": 1.109092354774475, "step": 4300, "unclipped_grad_norm": 0.7825598120689392 }, { "bits_per_byte": 1.6771565810531184, "learning_rate": 0.00015886360837076156, "loss_nats": 1.1625163555145264, "step": 4310, "unclipped_grad_norm": 0.7940689921379089 }, { "bits_per_byte": 1.6577068910843924, "learning_rate": 0.00015830619311911495, "loss_nats": 1.149034857749939, "step": 4320, "unclipped_grad_norm": 0.7976187467575073 }, { "bits_per_byte": 1.6195226150092283, "learning_rate": 0.00015774889224615915, "loss_nats": 1.1225675344467163, "step": 4330, "unclipped_grad_norm": 0.7892199158668518 }, { "bits_per_byte": 1.655279183984813, "learning_rate": 0.00015719171527462834, "loss_nats": 1.1473520994186401, "step": 4340, "unclipped_grad_norm": 0.8144169449806213 }, { "bits_per_byte": 1.6394932404274176, "learning_rate": 0.0001566346717251397, "loss_nats": 1.136410117149353, "step": 4350, "unclipped_grad_norm": 0.8119622468948364 }, { "bits_per_byte": 1.6287904160989233, "learning_rate": 0.00015607777111603056, "loss_nats": 1.1289914846420288, "step": 4360, "unclipped_grad_norm": 0.7554351091384888 }, { "bits_per_byte": 1.6321829458698431, "learning_rate": 0.00015552102296319576, "loss_nats": 1.1313430070877075, "step": 4370, "unclipped_grad_norm": 0.808085024356842 }, { "bits_per_byte": 1.6820371047193032, "learning_rate": 0.00015496443677992516, "loss_nats": 1.1658992767333984, "step": 4380, "unclipped_grad_norm": 0.7708722949028015 }, { "bits_per_byte": 1.6522423143358775, "learning_rate": 0.0001544080220767409, "loss_nats": 1.1452471017837524, "step": 4390, "unclipped_grad_norm": 0.8123121857643127 }, { "bits_per_byte": 1.6025224739367414, "learning_rate": 0.00015385178836123511, "loss_nats": 1.1107839345932007, "step": 4400, "unclipped_grad_norm": 0.7717524766921997 }, { "bits_per_byte": 1.6853694405624209, "learning_rate": 0.00015329574513790726, "loss_nats": 1.1682090759277344, "step": 4410, "unclipped_grad_norm": 0.81525057554245 }, { "bits_per_byte": 1.6142167782474597, "learning_rate": 0.00015273990190800186, "loss_nats": 1.1188898086547852, "step": 4420, "unclipped_grad_norm": 0.7956154346466064 }, { "bits_per_byte": 1.6755413199962148, "learning_rate": 0.0001521842681693461, "loss_nats": 1.1613967418670654, "step": 4430, "unclipped_grad_norm": 0.7907513380050659 }, { "bits_per_byte": 1.6011154838700328, "learning_rate": 0.00015162885341618752, "loss_nats": 1.1098086833953857, "step": 4440, "unclipped_grad_norm": 0.7607184648513794 }, { "bits_per_byte": 1.5990663105850023, "learning_rate": 0.00015107366713903178, "loss_nats": 1.1083883047103882, "step": 4450, "unclipped_grad_norm": 0.7442869544029236 }, { "bits_per_byte": 1.5859970049806358, "learning_rate": 0.0001505187188244806, "loss_nats": 1.0993293523788452, "step": 4460, "unclipped_grad_norm": 0.7537500262260437 }, { "bits_per_byte": 1.6451918855637568, "learning_rate": 0.00014996401795506952, "loss_nats": 1.1403601169586182, "step": 4470, "unclipped_grad_norm": 0.7931647896766663 }, { "bits_per_byte": 1.6505943765753102, "learning_rate": 0.00014940957400910585, "loss_nats": 1.1441048383712769, "step": 4480, "unclipped_grad_norm": 0.9106901288032532 }, { "bits_per_byte": 1.6301040195862135, "learning_rate": 0.00014885539646050699, "loss_nats": 1.1299020051956177, "step": 4490, "unclipped_grad_norm": 0.7690432667732239 }, { "bits_per_byte": 1.6520481459230536, "learning_rate": 0.00014830149477863805, "loss_nats": 1.1451125144958496, "step": 4500, "unclipped_grad_norm": 0.7990603446960449 }, { "bits_per_byte": 1.5968599451570833, "learning_rate": 0.00014774787842815066, "loss_nats": 1.1068589687347412, "step": 4510, "unclipped_grad_norm": 0.8130239248275757 }, { "bits_per_byte": 1.6430763269754933, "learning_rate": 0.00014719455686882066, "loss_nats": 1.138893723487854, "step": 4520, "unclipped_grad_norm": 0.7745712399482727 }, { "bits_per_byte": 1.660709880151108, "learning_rate": 0.00014664153955538695, "loss_nats": 1.1511163711547852, "step": 4530, "unclipped_grad_norm": 0.7695059776306152 }, { "bits_per_byte": 1.707295852686222, "learning_rate": 0.00014608883593738946, "loss_nats": 1.1834073066711426, "step": 4540, "unclipped_grad_norm": 0.8436031937599182 }, { "bits_per_byte": 1.5623325361872151, "learning_rate": 0.00014553645545900817, "loss_nats": 1.0829263925552368, "step": 4550, "unclipped_grad_norm": 0.7549967169761658 }, { "bits_per_byte": 1.6257084869954619, "learning_rate": 0.0001449844075589013, "loss_nats": 1.1268552541732788, "step": 4560, "unclipped_grad_norm": 0.8449703454971313 }, { "bits_per_byte": 1.6175644205465043, "learning_rate": 0.0001444327016700444, "loss_nats": 1.1212102174758911, "step": 4570, "unclipped_grad_norm": 0.7528937458992004 }, { "bits_per_byte": 1.570274179056107, "learning_rate": 0.0001438813472195688, "loss_nats": 1.0884311199188232, "step": 4580, "unclipped_grad_norm": 0.790579617023468 }, { "bits_per_byte": 1.609637052237634, "learning_rate": 0.00014333035362860093, "loss_nats": 1.1157153844833374, "step": 4590, "unclipped_grad_norm": 0.7935193181037903 }, { "bits_per_byte": 1.6088013885370918, "learning_rate": 0.00014277973031210088, "loss_nats": 1.1151361465454102, "step": 4600, "unclipped_grad_norm": 0.7915687561035156 }, { "bits_per_byte": 1.6615916352020816, "learning_rate": 0.00014222948667870207, "loss_nats": 1.151727557182312, "step": 4610, "unclipped_grad_norm": 0.8493216633796692 }, { "bits_per_byte": 1.6675298521710726, "learning_rate": 0.00014167963213054991, "loss_nats": 1.1558436155319214, "step": 4620, "unclipped_grad_norm": 0.8179065585136414 }, { "bits_per_byte": 1.6286588493710132, "learning_rate": 0.00014113017606314152, "loss_nats": 1.1289002895355225, "step": 4630, "unclipped_grad_norm": 0.8027536273002625 }, { "bits_per_byte": 1.7031861552612044, "learning_rate": 0.00014058112786516518, "loss_nats": 1.180558681488037, "step": 4640, "unclipped_grad_norm": 0.8179929256439209 }, { "bits_per_byte": 1.6059254946493637, "learning_rate": 0.0001400324969183396, "loss_nats": 1.113142728805542, "step": 4650, "unclipped_grad_norm": 0.8228299021720886 }, { "bits_per_byte": 1.5739184914278899, "learning_rate": 0.00013948429259725406, "loss_nats": 1.0909571647644043, "step": 4660, "unclipped_grad_norm": 0.820624589920044 }, { "bits_per_byte": 1.7237126286069824, "learning_rate": 0.00013893652426920776, "loss_nats": 1.194786548614502, "step": 4670, "unclipped_grad_norm": 0.8582949638366699 }, { "bits_per_byte": 1.7009104808249877, "learning_rate": 0.00013838920129405016, "loss_nats": 1.1789813041687012, "step": 4680, "unclipped_grad_norm": 0.8158925175666809 }, { "bits_per_byte": 1.6679322915740917, "learning_rate": 0.0001378423330240207, "loss_nats": 1.1561225652694702, "step": 4690, "unclipped_grad_norm": 0.849236249923706 }, { "bits_per_byte": 1.589747430648001, "learning_rate": 0.00013729592880358936, "loss_nats": 1.101928949356079, "step": 4700, "unclipped_grad_norm": 0.7403190732002258 }, { "bits_per_byte": 1.6742342518496576, "learning_rate": 0.00013674999796929659, "loss_nats": 1.1604907512664795, "step": 4710, "unclipped_grad_norm": 0.8582251071929932 }, { "bits_per_byte": 1.607032031025015, "learning_rate": 0.00013620454984959417, "loss_nats": 1.1139097213745117, "step": 4720, "unclipped_grad_norm": 0.8400667905807495 }, { "bits_per_byte": 1.5176990826873464, "learning_rate": 0.00013565959376468542, "loss_nats": 1.0519888401031494, "step": 4730, "unclipped_grad_norm": 0.7827285528182983 }, { "bits_per_byte": 1.5868581221135059, "learning_rate": 0.00013511513902636634, "loss_nats": 1.099926233291626, "step": 4740, "unclipped_grad_norm": 0.7815871238708496 }, { "bits_per_byte": 1.6377919880450826, "learning_rate": 0.0001345711949378662, "loss_nats": 1.1352308988571167, "step": 4750, "unclipped_grad_norm": 0.7846289873123169 }, { "bits_per_byte": 1.6125860387519786, "learning_rate": 0.00013402777079368853, "loss_nats": 1.1177594661712646, "step": 4760, "unclipped_grad_norm": 0.8307337164878845 }, { "bits_per_byte": 1.5936545325103024, "learning_rate": 0.0001334848758794528, "loss_nats": 1.1046371459960938, "step": 4770, "unclipped_grad_norm": 0.8213374614715576 }, { "bits_per_byte": 1.6121588338472355, "learning_rate": 0.00013294251947173504, "loss_nats": 1.1174633502960205, "step": 4780, "unclipped_grad_norm": 0.8224621415138245 }, { "bits_per_byte": 1.5705015401205475, "learning_rate": 0.00013240071083790996, "loss_nats": 1.0885887145996094, "step": 4790, "unclipped_grad_norm": 0.8122685551643372 }, { "bits_per_byte": 1.5722774329733569, "learning_rate": 0.0001318594592359921, "loss_nats": 1.0898196697235107, "step": 4800, "unclipped_grad_norm": 0.8455460667610168 }, { "bits_per_byte": 1.6480244398234671, "learning_rate": 0.0001313187739144781, "loss_nats": 1.1423234939575195, "step": 4810, "unclipped_grad_norm": 0.8817458748817444 }, { "bits_per_byte": 1.574972745077679, "learning_rate": 0.0001307786641121882, "loss_nats": 1.0916879177093506, "step": 4820, "unclipped_grad_norm": 0.7954387664794922 }, { "bits_per_byte": 1.608869665649484, "learning_rate": 0.00013023913905810876, "loss_nats": 1.1151834726333618, "step": 4830, "unclipped_grad_norm": 0.8687117695808411 }, { "bits_per_byte": 1.5985519104762718, "learning_rate": 0.00012970020797123432, "loss_nats": 1.1080317497253418, "step": 4840, "unclipped_grad_norm": 0.8505465388298035 }, { "bits_per_byte": 1.6228387844831649, "learning_rate": 0.00012916188006041027, "loss_nats": 1.1248661279678345, "step": 4850, "unclipped_grad_norm": 0.7912812232971191 }, { "bits_per_byte": 1.5432041098103009, "learning_rate": 0.00012862416452417516, "loss_nats": 1.0696675777435303, "step": 4860, "unclipped_grad_norm": 0.7933070063591003 }, { "bits_per_byte": 1.635918581029234, "learning_rate": 0.00012808707055060405, "loss_nats": 1.13393235206604, "step": 4870, "unclipped_grad_norm": 0.8805916905403137 }, { "bits_per_byte": 1.4766798445953597, "learning_rate": 0.000127550607317151, "loss_nats": 1.0235564708709717, "step": 4880, "unclipped_grad_norm": 0.8066675066947937 }, { "bits_per_byte": 1.5344280071194227, "learning_rate": 0.00012701478399049248, "loss_nats": 1.0635844469070435, "step": 4890, "unclipped_grad_norm": 0.8088182806968689 }, { "bits_per_byte": 1.6101516243290155, "learning_rate": 0.0001264796097263708, "loss_nats": 1.1160720586776733, "step": 4900, "unclipped_grad_norm": 0.8653903603553772 }, { "bits_per_byte": 1.6779198400576476, "learning_rate": 0.0001259450936694375, "loss_nats": 1.1630454063415527, "step": 4910, "unclipped_grad_norm": 0.8391038179397583 }, { "bits_per_byte": 1.6284904783558185, "learning_rate": 0.00012541124495309727, "loss_nats": 1.1287835836410522, "step": 4920, "unclipped_grad_norm": 0.8723531365394592 }, { "bits_per_byte": 1.5831391692712484, "learning_rate": 0.00012487807269935167, "loss_nats": 1.0973484516143799, "step": 4930, "unclipped_grad_norm": 0.7553638815879822 }, { "bits_per_byte": 1.5755332365368409, "learning_rate": 0.00012434558601864349, "loss_nats": 1.0920764207839966, "step": 4940, "unclipped_grad_norm": 0.8158645033836365 }, { "bits_per_byte": 1.530142715407874, "learning_rate": 0.0001238137940097009, "loss_nats": 1.0606141090393066, "step": 4950, "unclipped_grad_norm": 0.7773033380508423 }, { "bits_per_byte": 1.5220387209165676, "learning_rate": 0.0001232827057593821, "loss_nats": 1.0549968481063843, "step": 4960, "unclipped_grad_norm": 0.8325244784355164 }, { "bits_per_byte": 1.6031579498316795, "learning_rate": 0.00012275233034251992, "loss_nats": 1.1112244129180908, "step": 4970, "unclipped_grad_norm": 0.8591333627700806 }, { "bits_per_byte": 1.6532000857185325, "learning_rate": 0.0001222226768217669, "loss_nats": 1.1459109783172607, "step": 4980, "unclipped_grad_norm": 0.8351151347160339 }, { "bits_per_byte": 1.585272098107249, "learning_rate": 0.00012169375424744034, "loss_nats": 1.0988268852233887, "step": 4990, "unclipped_grad_norm": 0.8779996037483215 }, { "bits_per_byte": 1.5394700224947675, "learning_rate": 0.00012116557165736772, "loss_nats": 1.0670793056488037, "step": 5000, "unclipped_grad_norm": 0.7928086519241333 }, { "bits_per_byte": 1.5552827953456976, "learning_rate": 0.00012063813807673214, "loss_nats": 1.0780398845672607, "step": 5010, "unclipped_grad_norm": 0.8201308846473694 }, { "bits_per_byte": 1.5898205232746179, "learning_rate": 0.00012011146251791832, "loss_nats": 1.1019796133041382, "step": 5020, "unclipped_grad_norm": 0.7774789929389954 }, { "bits_per_byte": 1.5697566832596608, "learning_rate": 0.0001195855539803584, "loss_nats": 1.088072419166565, "step": 5030, "unclipped_grad_norm": 0.7855319380760193 }, { "bits_per_byte": 1.539816567536256, "learning_rate": 0.00011906042145037817, "loss_nats": 1.0673195123672485, "step": 5040, "unclipped_grad_norm": 0.7920348048210144 }, { "bits_per_byte": 1.5092143186070297, "learning_rate": 0.0001185360739010438, "loss_nats": 1.0461076498031616, "step": 5050, "unclipped_grad_norm": 0.8045132160186768 }, { "bits_per_byte": 1.5339026001210367, "learning_rate": 0.00011801252029200808, "loss_nats": 1.0632202625274658, "step": 5060, "unclipped_grad_norm": 0.8044583797454834 }, { "bits_per_byte": 1.6138325690054323, "learning_rate": 0.00011748976956935777, "loss_nats": 1.1186234951019287, "step": 5070, "unclipped_grad_norm": 0.8723475337028503 }, { "bits_per_byte": 1.574335549356232, "learning_rate": 0.00011696783066546024, "loss_nats": 1.091246247291565, "step": 5080, "unclipped_grad_norm": 0.8741123080253601 }, { "bits_per_byte": 1.5594206979254568, "learning_rate": 0.00011644671249881147, "loss_nats": 1.0809080600738525, "step": 5090, "unclipped_grad_norm": 0.8465037941932678 }, { "bits_per_byte": 1.5640659493252613, "learning_rate": 0.00011592642397388293, "loss_nats": 1.0841279029846191, "step": 5100, "unclipped_grad_norm": 0.783299446105957 }, { "bits_per_byte": 1.488738924108558, "learning_rate": 0.00011540697398097015, "loss_nats": 1.0319151878356934, "step": 5110, "unclipped_grad_norm": 0.8348572254180908 }, { "bits_per_byte": 1.5847128105266433, "learning_rate": 0.00011488837139604016, "loss_nats": 1.0984392166137695, "step": 5120, "unclipped_grad_norm": 0.8110998868942261 }, { "bits_per_byte": 1.590178763136365, "learning_rate": 0.00011437062508058039, "loss_nats": 1.1022279262542725, "step": 5130, "unclipped_grad_norm": 0.8208310604095459 }, { "bits_per_byte": 1.5494388248693793, "learning_rate": 0.00011385374388144673, "loss_nats": 1.0739891529083252, "step": 5140, "unclipped_grad_norm": 0.8678956627845764 }, { "bits_per_byte": 1.6273239200350162, "learning_rate": 0.00011333773663071286, "loss_nats": 1.1279749870300293, "step": 5150, "unclipped_grad_norm": 0.848126232624054 }, { "bits_per_byte": 1.5085432423033631, "learning_rate": 0.00011282261214551897, "loss_nats": 1.0456424951553345, "step": 5160, "unclipped_grad_norm": 0.7922812104225159 }, { "bits_per_byte": 1.6700107019097687, "learning_rate": 0.00011230837922792112, "loss_nats": 1.1575632095336914, "step": 5170, "unclipped_grad_norm": 0.8759046196937561 }, { "bits_per_byte": 1.5869253673299932, "learning_rate": 0.0001117950466647412, "loss_nats": 1.0999728441238403, "step": 5180, "unclipped_grad_norm": 0.8647724390029907 }, { "bits_per_byte": 1.5237559676854329, "learning_rate": 0.00011128262322741627, "loss_nats": 1.0561871528625488, "step": 5190, "unclipped_grad_norm": 0.813337504863739 }, { "bits_per_byte": 1.556985939537192, "learning_rate": 0.00011077111767184913, "loss_nats": 1.0792204141616821, "step": 5200, "unclipped_grad_norm": 0.8388134241104126 }, { "bits_per_byte": 1.5602121620847276, "learning_rate": 0.00011026053873825837, "loss_nats": 1.0814566612243652, "step": 5210, "unclipped_grad_norm": 0.8162683248519897 }, { "bits_per_byte": 1.6432956048553435, "learning_rate": 0.0001097508951510293, "loss_nats": 1.1390457153320312, "step": 5220, "unclipped_grad_norm": 0.8667425513267517 }, { "bits_per_byte": 1.5501931407760636, "learning_rate": 0.0001092421956185646, "loss_nats": 1.074512004852295, "step": 5230, "unclipped_grad_norm": 0.8668702840805054 }, { "bits_per_byte": 1.5510165937083948, "learning_rate": 0.00010873444883313581, "loss_nats": 1.075082778930664, "step": 5240, "unclipped_grad_norm": 0.8049055337905884 }, { "bits_per_byte": 1.5393878147876552, "learning_rate": 0.00010822766347073447, "loss_nats": 1.0670223236083984, "step": 5250, "unclipped_grad_norm": 0.8209428191184998 }, { "bits_per_byte": 1.540701762240247, "learning_rate": 0.00010772184819092424, "loss_nats": 1.0679330825805664, "step": 5260, "unclipped_grad_norm": 0.8524174690246582 }, { "bits_per_byte": 1.5376201771020872, "learning_rate": 0.0001072170116366926, "loss_nats": 1.0657970905303955, "step": 5270, "unclipped_grad_norm": 0.8064137697219849 }, { "bits_per_byte": 1.5550475230793173, "learning_rate": 0.00010671316243430345, "loss_nats": 1.0778768062591553, "step": 5280, "unclipped_grad_norm": 0.8169121146202087 }, { "bits_per_byte": 1.5423493560355128, "learning_rate": 0.00010621030919314944, "loss_nats": 1.069075107574463, "step": 5290, "unclipped_grad_norm": 0.8263237476348877 }, { "bits_per_byte": 1.5542627662434303, "learning_rate": 0.000105708460505605, "loss_nats": 1.077332854270935, "step": 5300, "unclipped_grad_norm": 0.8492859601974487 }, { "bits_per_byte": 1.5509104804128124, "learning_rate": 0.00010520762494687964, "loss_nats": 1.0750092267990112, "step": 5310, "unclipped_grad_norm": 0.8885292410850525 }, { "bits_per_byte": 1.5517654061702515, "learning_rate": 0.00010470781107487111, "loss_nats": 1.0756018161773682, "step": 5320, "unclipped_grad_norm": 0.8326203227043152 }, { "bits_per_byte": 1.5477100552629064, "learning_rate": 0.00010420902743001955, "loss_nats": 1.0727908611297607, "step": 5330, "unclipped_grad_norm": 0.8806997537612915 }, { "bits_per_byte": 1.503639328996661, "learning_rate": 0.00010371128253516117, "loss_nats": 1.0422433614730835, "step": 5340, "unclipped_grad_norm": 0.8234722018241882 }, { "bits_per_byte": 1.5869049013945407, "learning_rate": 0.000103214584895383, "loss_nats": 1.0999586582183838, "step": 5350, "unclipped_grad_norm": 0.8706242442131042 }, { "bits_per_byte": 1.5072631754329913, "learning_rate": 0.00010271894299787715, "loss_nats": 1.044755220413208, "step": 5360, "unclipped_grad_norm": 0.8708282709121704 }, { "bits_per_byte": 1.471025742965594, "learning_rate": 0.00010222436531179622, "loss_nats": 1.0196373462677002, "step": 5370, "unclipped_grad_norm": 0.8303158283233643 }, { "bits_per_byte": 1.5968417149960918, "learning_rate": 0.0001017308602881082, "loss_nats": 1.1068463325500488, "step": 5380, "unclipped_grad_norm": 0.883955180644989 }, { "bits_per_byte": 1.531507741707772, "learning_rate": 0.00010123843635945242, "loss_nats": 1.0615602731704712, "step": 5390, "unclipped_grad_norm": 0.8507125377655029 }, { "bits_per_byte": 1.5483677169198058, "learning_rate": 0.00010074710193999505, "loss_nats": 1.073246717453003, "step": 5400, "unclipped_grad_norm": 0.8522453904151917 }, { "bits_per_byte": 1.571479089508052, "learning_rate": 0.0001002568654252858, "loss_nats": 1.089266300201416, "step": 5410, "unclipped_grad_norm": 0.8313266038894653 }, { "bits_per_byte": 1.5408727129952047, "learning_rate": 9.976773519211404e-05, "loss_nats": 1.0680515766143799, "step": 5420, "unclipped_grad_norm": 0.8902517557144165 }, { "bits_per_byte": 1.5778336764747818, "learning_rate": 9.92797195983659e-05, "loss_nats": 1.0936709642410278, "step": 5430, "unclipped_grad_norm": 0.8956695795059204 }, { "bits_per_byte": 1.513927503153925, "learning_rate": 9.879282698288142e-05, "loss_nats": 1.0493745803833008, "step": 5440, "unclipped_grad_norm": 0.897573709487915 }, { "bits_per_byte": 1.5078418970531444, "learning_rate": 9.830706566531197e-05, "loss_nats": 1.0451563596725464, "step": 5450, "unclipped_grad_norm": 0.8440007567405701 }, { "bits_per_byte": 1.5030907043404085, "learning_rate": 9.782244394597826e-05, "loss_nats": 1.0418630838394165, "step": 5460, "unclipped_grad_norm": 0.8436890244483948 }, { "bits_per_byte": 1.5464133060754008, "learning_rate": 9.733897010572829e-05, "loss_nats": 1.0718920230865479, "step": 5470, "unclipped_grad_norm": 0.8681005239486694 }, { "bits_per_byte": 1.553998600891705, "learning_rate": 9.685665240579612e-05, "loss_nats": 1.077149748802185, "step": 5480, "unclipped_grad_norm": 0.8523774147033691 }, { "bits_per_byte": 1.543775436176467, "learning_rate": 9.637549908766041e-05, "loss_nats": 1.070063591003418, "step": 5490, "unclipped_grad_norm": 0.8906207084655762 }, { "bits_per_byte": 1.5444891641775478, "learning_rate": 9.589551837290389e-05, "loss_nats": 1.0705583095550537, "step": 5500, "unclipped_grad_norm": 0.8658390045166016 }, { "bits_per_byte": 1.6370650173798855, "learning_rate": 9.541671846307257e-05, "loss_nats": 1.1347270011901855, "step": 5510, "unclipped_grad_norm": 0.9161565899848938 }, { "bits_per_byte": 1.5520985365649729, "learning_rate": 9.493910753953597e-05, "loss_nats": 1.075832724571228, "step": 5520, "unclipped_grad_norm": 0.845907986164093 }, { "bits_per_byte": 1.5215033389244317, "learning_rate": 9.446269376334689e-05, "loss_nats": 1.0546257495880127, "step": 5530, "unclipped_grad_norm": 0.8585748672485352 }, { "bits_per_byte": 1.5943249208833656, "learning_rate": 9.39874852751024e-05, "loss_nats": 1.1051018238067627, "step": 5540, "unclipped_grad_norm": 0.9360165596008301 }, { "bits_per_byte": 1.5176287417831436, "learning_rate": 9.351349019480433e-05, "loss_nats": 1.0519400835037231, "step": 5550, "unclipped_grad_norm": 0.8308430910110474 }, { "bits_per_byte": 1.5164790377621256, "learning_rate": 9.304071662172086e-05, "loss_nats": 1.0511431694030762, "step": 5560, "unclipped_grad_norm": 0.894058346748352 }, { "bits_per_byte": 1.4905304673825959, "learning_rate": 9.256917263424787e-05, "loss_nats": 1.0331569910049438, "step": 5570, "unclipped_grad_norm": 0.8399901390075684 }, { "bits_per_byte": 1.5655759569998369, "learning_rate": 9.209886628977114e-05, "loss_nats": 1.085174560546875, "step": 5580, "unclipped_grad_norm": 0.8579773902893066 }, { "bits_per_byte": 1.482571282283316, "learning_rate": 9.162980562452843e-05, "loss_nats": 1.0276401042938232, "step": 5590, "unclipped_grad_norm": 0.8161733746528625 }, { "bits_per_byte": 1.5496305855250914, "learning_rate": 9.116199865347238e-05, "loss_nats": 1.0741220712661743, "step": 5600, "unclipped_grad_norm": 0.8440633416175842 }, { "bits_per_byte": 1.5466313800766949, "learning_rate": 9.069545337013333e-05, "loss_nats": 1.0720431804656982, "step": 5610, "unclipped_grad_norm": 0.8384022116661072 }, { "bits_per_byte": 1.55366271877457, "learning_rate": 9.0230177746483e-05, "loss_nats": 1.0769169330596924, "step": 5620, "unclipped_grad_norm": 0.8643615245819092 }, { "bits_per_byte": 1.5359743031333302, "learning_rate": 8.976617973279797e-05, "loss_nats": 1.0646562576293945, "step": 5630, "unclipped_grad_norm": 0.8559057712554932 }, { "bits_per_byte": 1.5154779267514533, "learning_rate": 8.930346725752419e-05, "loss_nats": 1.050449252128601, "step": 5640, "unclipped_grad_norm": 0.8564271330833435 }, { "bits_per_byte": 1.5399061704973558, "learning_rate": 8.88420482271411e-05, "loss_nats": 1.0673816204071045, "step": 5650, "unclipped_grad_norm": 0.899230420589447 }, { "bits_per_byte": 1.6017691899259623, "learning_rate": 8.838193052602691e-05, "loss_nats": 1.1102617979049683, "step": 5660, "unclipped_grad_norm": 0.8796955943107605 }, { "bits_per_byte": 1.5203328250026595, "learning_rate": 8.792312201632352e-05, "loss_nats": 1.05381441116333, "step": 5670, "unclipped_grad_norm": 0.8509314060211182 }, { "bits_per_byte": 1.5301421994599214, "learning_rate": 8.746563053780261e-05, "loss_nats": 1.060613751411438, "step": 5680, "unclipped_grad_norm": 0.8624072074890137 }, { "bits_per_byte": 1.6028342784827556, "learning_rate": 8.700946390773114e-05, "loss_nats": 1.1110000610351562, "step": 5690, "unclipped_grad_norm": 0.8914096355438232 }, { "bits_per_byte": 1.526354281574668, "learning_rate": 8.655462992073833e-05, "loss_nats": 1.057988166809082, "step": 5700, "unclipped_grad_norm": 0.8579604625701904 }, { "bits_per_byte": 1.5347007716036911, "learning_rate": 8.610113634868199e-05, "loss_nats": 1.063773512840271, "step": 5710, "unclipped_grad_norm": 0.8810272216796875 }, { "bits_per_byte": 1.5491378552303696, "learning_rate": 8.564899094051613e-05, "loss_nats": 1.0737805366516113, "step": 5720, "unclipped_grad_norm": 0.8394830226898193 }, { "bits_per_byte": 1.5456280332915613, "learning_rate": 8.519820142215816e-05, "loss_nats": 1.071347713470459, "step": 5730, "unclipped_grad_norm": 0.865943968296051 }, { "bits_per_byte": 1.5433466834278662, "learning_rate": 8.474877549635728e-05, "loss_nats": 1.0697664022445679, "step": 5740, "unclipped_grad_norm": 0.886944591999054 }, { "bits_per_byte": 1.5488365416260579, "learning_rate": 8.430072084256247e-05, "loss_nats": 1.0735716819763184, "step": 5750, "unclipped_grad_norm": 0.9145897626876831 }, { "bits_per_byte": 1.5636783004302166, "learning_rate": 8.385404511679161e-05, "loss_nats": 1.0838592052459717, "step": 5760, "unclipped_grad_norm": 0.9009037017822266 }, { "bits_per_byte": 1.511110599315443, "learning_rate": 8.340875595150034e-05, "loss_nats": 1.0474220514297485, "step": 5770, "unclipped_grad_norm": 0.844070315361023 }, { "bits_per_byte": 1.494012772097266, "learning_rate": 8.296486095545203e-05, "loss_nats": 1.035570740699768, "step": 5780, "unclipped_grad_norm": 0.8943845629692078 }, { "bits_per_byte": 1.5135893852623288, "learning_rate": 8.252236771358729e-05, "loss_nats": 1.049140214920044, "step": 5790, "unclipped_grad_norm": 0.8841454386711121 }, { "bits_per_byte": 1.4905120652389536, "learning_rate": 8.208128378689484e-05, "loss_nats": 1.033144235610962, "step": 5800, "unclipped_grad_norm": 0.8580894470214844 }, { "bits_per_byte": 1.5735937881830608, "learning_rate": 8.16416167122819e-05, "loss_nats": 1.0907320976257324, "step": 5810, "unclipped_grad_norm": 0.933175265789032 }, { "bits_per_byte": 1.5042643139498963, "learning_rate": 8.120337400244578e-05, "loss_nats": 1.042676568031311, "step": 5820, "unclipped_grad_norm": 0.828717052936554 }, { "bits_per_byte": 1.5156406223391694, "learning_rate": 8.076656314574521e-05, "loss_nats": 1.0505620241165161, "step": 5830, "unclipped_grad_norm": 0.9511673450469971 }, { "bits_per_byte": 1.5092205099824607, "learning_rate": 8.03311916060725e-05, "loss_nats": 1.0461119413375854, "step": 5840, "unclipped_grad_norm": 0.8395604491233826 }, { "bits_per_byte": 1.477936693807865, "learning_rate": 7.989726682272606e-05, "loss_nats": 1.0244276523590088, "step": 5850, "unclipped_grad_norm": 0.9143134355545044 }, { "bits_per_byte": 1.5336375748560571, "learning_rate": 7.946479621028318e-05, "loss_nats": 1.063036561012268, "step": 5860, "unclipped_grad_norm": 0.9014923572540283 }, { "bits_per_byte": 1.5402322496033916, "learning_rate": 7.903378715847343e-05, "loss_nats": 1.0676076412200928, "step": 5870, "unclipped_grad_norm": 0.8728002905845642 }, { "bits_per_byte": 1.555285031120159, "learning_rate": 7.860424703205224e-05, "loss_nats": 1.078041434288025, "step": 5880, "unclipped_grad_norm": 0.8936865925788879 }, { "bits_per_byte": 1.564054942435606, "learning_rate": 7.817618317067534e-05, "loss_nats": 1.084120273590088, "step": 5890, "unclipped_grad_norm": 0.8752698302268982 }, { "bits_per_byte": 1.5458715607251832, "learning_rate": 7.774960288877295e-05, "loss_nats": 1.071516513824463, "step": 5900, "unclipped_grad_norm": 0.8957662582397461 }, { "bits_per_byte": 1.5568008862048637, "learning_rate": 7.732451347542522e-05, "loss_nats": 1.0790921449661255, "step": 5910, "unclipped_grad_norm": 0.8808028101921082 }, { "bits_per_byte": 1.4896764015384112, "learning_rate": 7.690092219423722e-05, "loss_nats": 1.0325649976730347, "step": 5920, "unclipped_grad_norm": 0.8660398721694946 }, { "bits_per_byte": 1.5908711652887386, "learning_rate": 7.647883628321536e-05, "loss_nats": 1.102707862854004, "step": 5930, "unclipped_grad_norm": 0.9604227542877197 }, { "bits_per_byte": 1.4574508083676907, "learning_rate": 7.605826295464316e-05, "loss_nats": 1.010227918624878, "step": 5940, "unclipped_grad_norm": 0.8380697965621948 }, { "bits_per_byte": 1.5131755950043528, "learning_rate": 7.563920939495851e-05, "loss_nats": 1.0488533973693848, "step": 5950, "unclipped_grad_norm": 0.8392192125320435 }, { "bits_per_byte": 1.5452194025131114, "learning_rate": 7.522168276463047e-05, "loss_nats": 1.0710644721984863, "step": 5960, "unclipped_grad_norm": 0.932554304599762 }, { "bits_per_byte": 1.5213482105733536, "learning_rate": 7.480569019803719e-05, "loss_nats": 1.054518222808838, "step": 5970, "unclipped_grad_norm": 0.927780270576477 }, { "bits_per_byte": 1.5601101763727658, "learning_rate": 7.439123880334388e-05, "loss_nats": 1.0813859701156616, "step": 5980, "unclipped_grad_norm": 0.8920019268989563 }, { "bits_per_byte": 1.5382338112003657, "learning_rate": 7.39783356623813e-05, "loss_nats": 1.0662224292755127, "step": 5990, "unclipped_grad_norm": 0.9537093639373779 }, { "bits_per_byte": 1.4675097296513546, "learning_rate": 7.356698783052497e-05, "loss_nats": 1.017200231552124, "step": 6000, "unclipped_grad_norm": 0.8773758411407471 }, { "bits_per_byte": 1.506333093257125, "learning_rate": 7.315720233657431e-05, "loss_nats": 1.0441105365753174, "step": 6010, "unclipped_grad_norm": 0.8785988092422485 }, { "bits_per_byte": 1.4794742187065786, "learning_rate": 7.274898618263286e-05, "loss_nats": 1.0254933834075928, "step": 6020, "unclipped_grad_norm": 0.8875186443328857 }, { "bits_per_byte": 1.5245675538148544, "learning_rate": 7.234234634398831e-05, "loss_nats": 1.056749701499939, "step": 6030, "unclipped_grad_norm": 0.88789963722229 }, { "bits_per_byte": 1.5628784091210537, "learning_rate": 7.193728976899362e-05, "loss_nats": 1.083304762840271, "step": 6040, "unclipped_grad_norm": 0.9149079322814941 }, { "bits_per_byte": 1.4881958028971334, "learning_rate": 7.153382337894797e-05, "loss_nats": 1.031538724899292, "step": 6050, "unclipped_grad_norm": 0.9299342632293701 }, { "bits_per_byte": 1.4785307218839452, "learning_rate": 7.113195406797887e-05, "loss_nats": 1.0248394012451172, "step": 6060, "unclipped_grad_norm": 0.8783867359161377 }, { "bits_per_byte": 1.5509557118499895, "learning_rate": 7.073168870292397e-05, "loss_nats": 1.075040578842163, "step": 6070, "unclipped_grad_norm": 0.9337034821510315 }, { "bits_per_byte": 1.4914187577743023, "learning_rate": 7.033303412321404e-05, "loss_nats": 1.0337727069854736, "step": 6080, "unclipped_grad_norm": 0.9420101046562195 }, { "bits_per_byte": 1.4968499698885498, "learning_rate": 6.993599714075586e-05, "loss_nats": 1.0375373363494873, "step": 6090, "unclipped_grad_norm": 0.8664546012878418 }, { "bits_per_byte": 1.4906561867003767, "learning_rate": 6.954058453981609e-05, "loss_nats": 1.0332441329956055, "step": 6100, "unclipped_grad_norm": 0.8687640428543091 }, { "bits_per_byte": 1.5538661742505406, "learning_rate": 6.914680307690504e-05, "loss_nats": 1.077057957649231, "step": 6110, "unclipped_grad_norm": 0.8986173868179321 }, { "bits_per_byte": 1.5476213122150613, "learning_rate": 6.875465948066139e-05, "loss_nats": 1.0727293491363525, "step": 6120, "unclipped_grad_norm": 0.9189077019691467 }, { "bits_per_byte": 1.4789527673091625, "learning_rate": 6.836416045173733e-05, "loss_nats": 1.0251319408416748, "step": 6130, "unclipped_grad_norm": 0.8851786851882935 }, { "bits_per_byte": 1.4768556108645414, "learning_rate": 6.797531266268377e-05, "loss_nats": 1.0236783027648926, "step": 6140, "unclipped_grad_norm": 0.9043483138084412 }, { "bits_per_byte": 1.552988202817886, "learning_rate": 6.758812275783663e-05, "loss_nats": 1.0764493942260742, "step": 6150, "unclipped_grad_norm": 0.9005390405654907 }, { "bits_per_byte": 1.3994871515820966, "learning_rate": 6.720259735320298e-05, "loss_nats": 0.970050573348999, "step": 6160, "unclipped_grad_norm": 0.8554620742797852 }, { "bits_per_byte": 1.5302097886417105, "learning_rate": 6.681874303634842e-05, "loss_nats": 1.0606606006622314, "step": 6170, "unclipped_grad_norm": 0.8954014778137207 }, { "bits_per_byte": 1.4938230752333643, "learning_rate": 6.643656636628402e-05, "loss_nats": 1.0354392528533936, "step": 6180, "unclipped_grad_norm": 0.892278790473938 }, { "bits_per_byte": 1.516749394489282, "learning_rate": 6.605607387335466e-05, "loss_nats": 1.05133056640625, "step": 6190, "unclipped_grad_norm": 0.9185703992843628 }, { "bits_per_byte": 1.5086576107661869, "learning_rate": 6.567727205912724e-05, "loss_nats": 1.0457217693328857, "step": 6200, "unclipped_grad_norm": 0.8927735090255737 }, { "bits_per_byte": 1.5088584865023946, "learning_rate": 6.530016739627959e-05, "loss_nats": 1.045861005783081, "step": 6210, "unclipped_grad_norm": 0.8906724452972412 }, { "bits_per_byte": 1.5118191678369979, "learning_rate": 6.492476632848993e-05, "loss_nats": 1.0479131937026978, "step": 6220, "unclipped_grad_norm": 0.9077256917953491 }, { "bits_per_byte": 1.4471287536282067, "learning_rate": 6.455107527032678e-05, "loss_nats": 1.0030732154846191, "step": 6230, "unclipped_grad_norm": 0.8839235305786133 }, { "bits_per_byte": 1.5076786855174757, "learning_rate": 6.417910060713926e-05, "loss_nats": 1.0450432300567627, "step": 6240, "unclipped_grad_norm": 0.9515005946159363 }, { "bits_per_byte": 1.5408861276419719, "learning_rate": 6.380884869494799e-05, "loss_nats": 1.0680608749389648, "step": 6250, "unclipped_grad_norm": 0.901561975479126 }, { "bits_per_byte": 1.4200397662908057, "learning_rate": 6.34403258603367e-05, "loss_nats": 0.9842965602874756, "step": 6260, "unclipped_grad_norm": 0.889621376991272 }, { "bits_per_byte": 1.5216213190229235, "learning_rate": 6.30735384003437e-05, "loss_nats": 1.0547075271606445, "step": 6270, "unclipped_grad_norm": 0.9411186575889587 }, { "bits_per_byte": 1.5224643779774532, "learning_rate": 6.270849258235484e-05, "loss_nats": 1.0552918910980225, "step": 6280, "unclipped_grad_norm": 0.9406285881996155 }, { "bits_per_byte": 1.452789390598705, "learning_rate": 6.234519464399581e-05, "loss_nats": 1.0069968700408936, "step": 6290, "unclipped_grad_norm": 0.8839805126190186 }, { "bits_per_byte": 1.5074908804627336, "learning_rate": 6.198365079302614e-05, "loss_nats": 1.0449130535125732, "step": 6300, "unclipped_grad_norm": 0.9184122681617737 }, { "bits_per_byte": 1.478450577968643, "learning_rate": 6.16238672072326e-05, "loss_nats": 1.0247838497161865, "step": 6310, "unclipped_grad_norm": 0.8948619365692139 }, { "bits_per_byte": 1.4214500240278807, "learning_rate": 6.126585003432406e-05, "loss_nats": 0.985274076461792, "step": 6320, "unclipped_grad_norm": 0.8749227523803711 }, { "bits_per_byte": 1.5185149683830395, "learning_rate": 6.090960539182614e-05, "loss_nats": 1.0525543689727783, "step": 6330, "unclipped_grad_norm": 0.9816636443138123 }, { "bits_per_byte": 1.4759477144506365, "learning_rate": 6.0555139366976935e-05, "loss_nats": 1.023048996925354, "step": 6340, "unclipped_grad_norm": 0.8950188159942627 }, { "bits_per_byte": 1.587707716408768, "learning_rate": 6.020245801662274e-05, "loss_nats": 1.1005151271820068, "step": 6350, "unclipped_grad_norm": 1.01009202003479 }, { "bits_per_byte": 1.4638088349874379, "learning_rate": 5.985156736711483e-05, "loss_nats": 1.0146349668502808, "step": 6360, "unclipped_grad_norm": 0.9816907644271851 }, { "bits_per_byte": 1.4382974445064014, "learning_rate": 5.950247341420618e-05, "loss_nats": 0.9969518184661865, "step": 6370, "unclipped_grad_norm": 0.8970034122467041 }, { "bits_per_byte": 1.4818757844432269, "learning_rate": 5.915518212294935e-05, "loss_nats": 1.0271580219268799, "step": 6380, "unclipped_grad_norm": 0.9058489799499512 }, { "bits_per_byte": 1.527050123380059, "learning_rate": 5.880969942759425e-05, "loss_nats": 1.0584704875946045, "step": 6390, "unclipped_grad_norm": 0.929339587688446 }, { "bits_per_byte": 1.5156593684481134, "learning_rate": 5.846603123148688e-05, "loss_nats": 1.0505750179290771, "step": 6400, "unclipped_grad_norm": 0.9009133577346802 }, { "bits_per_byte": 1.4969940913499729, "learning_rate": 5.812418340696858e-05, "loss_nats": 1.0376372337341309, "step": 6410, "unclipped_grad_norm": 0.957128643989563 }, { "bits_per_byte": 1.4783625228514012, "learning_rate": 5.778416179527541e-05, "loss_nats": 1.0247228145599365, "step": 6420, "unclipped_grad_norm": 0.9414587616920471 }, { "bits_per_byte": 1.523227980947284, "learning_rate": 5.7445972206438635e-05, "loss_nats": 1.055821180343628, "step": 6430, "unclipped_grad_norm": 0.9234302639961243 }, { "bits_per_byte": 1.4623229048839832, "learning_rate": 5.710962041918516e-05, "loss_nats": 1.013604998588562, "step": 6440, "unclipped_grad_norm": 0.9177728295326233 }, { "bits_per_byte": 1.5306256426914968, "learning_rate": 5.677511218083908e-05, "loss_nats": 1.0609488487243652, "step": 6450, "unclipped_grad_norm": 0.9453822374343872 }, { "bits_per_byte": 1.5847898587542297, "learning_rate": 5.644245320722316e-05, "loss_nats": 1.0984926223754883, "step": 6460, "unclipped_grad_norm": 0.9647909998893738 }, { "bits_per_byte": 1.5690988496201106, "learning_rate": 5.61116491825615e-05, "loss_nats": 1.0876164436340332, "step": 6470, "unclipped_grad_norm": 0.9592546820640564 }, { "bits_per_byte": 1.4544314809491432, "learning_rate": 5.5782705759382104e-05, "loss_nats": 1.0081350803375244, "step": 6480, "unclipped_grad_norm": 0.9334172010421753 }, { "bits_per_byte": 1.5573938823850386, "learning_rate": 5.5455628558420546e-05, "loss_nats": 1.0795031785964966, "step": 6490, "unclipped_grad_norm": 0.967822790145874 }, { "bits_per_byte": 1.5280063469188563, "learning_rate": 5.513042316852368e-05, "loss_nats": 1.0591332912445068, "step": 6500, "unclipped_grad_norm": 0.9625112414360046 }, { "bits_per_byte": 1.4687796495453256, "learning_rate": 5.480709514655443e-05, "loss_nats": 1.018080472946167, "step": 6510, "unclipped_grad_norm": 0.9510544538497925 }, { "bits_per_byte": 1.4284500618832987, "learning_rate": 5.448565001729653e-05, "loss_nats": 0.9901261329650879, "step": 6520, "unclipped_grad_norm": 0.8985839486122131 }, { "bits_per_byte": 1.4546447394362132, "learning_rate": 5.41660932733604e-05, "loss_nats": 1.0082828998565674, "step": 6530, "unclipped_grad_norm": 0.907886803150177 }, { "bits_per_byte": 1.4970680438898438, "learning_rate": 5.384843037508902e-05, "loss_nats": 1.0376884937286377, "step": 6540, "unclipped_grad_norm": 0.9381445050239563 }, { "bits_per_byte": 1.4280130539674563, "learning_rate": 5.353266675046498e-05, "loss_nats": 0.9898232221603394, "step": 6550, "unclipped_grad_norm": 0.9031082391738892 }, { "bits_per_byte": 1.5210671909218436, "learning_rate": 5.321880779501729e-05, "loss_nats": 1.054323434829712, "step": 6560, "unclipped_grad_norm": 0.9433532357215881 }, { "bits_per_byte": 1.479243074023819, "learning_rate": 5.290685887172962e-05, "loss_nats": 1.0253331661224365, "step": 6570, "unclipped_grad_norm": 0.9311387538909912 }, { "bits_per_byte": 1.5051876888023787, "learning_rate": 5.2596825310948295e-05, "loss_nats": 1.0433166027069092, "step": 6580, "unclipped_grad_norm": 0.9569925665855408 }, { "bits_per_byte": 1.494823842278735, "learning_rate": 5.228871241029154e-05, "loss_nats": 1.0361329317092896, "step": 6590, "unclipped_grad_norm": 0.8958868384361267 }, { "bits_per_byte": 1.4302109062541577, "learning_rate": 5.198252543455865e-05, "loss_nats": 0.9913466572761536, "step": 6600, "unclipped_grad_norm": 0.9136191010475159 }, { "bits_per_byte": 1.436879103584736, "learning_rate": 5.167826961564027e-05, "loss_nats": 0.9959686994552612, "step": 6610, "unclipped_grad_norm": 0.8710504770278931 }, { "bits_per_byte": 1.4278971376607748, "learning_rate": 5.137595015242884e-05, "loss_nats": 0.9897428750991821, "step": 6620, "unclipped_grad_norm": 0.9145034551620483 }, { "bits_per_byte": 1.4582168190946336, "learning_rate": 5.1075572210729925e-05, "loss_nats": 1.010758876800537, "step": 6630, "unclipped_grad_norm": 0.8999127149581909 }, { "bits_per_byte": 1.5240701799885592, "learning_rate": 5.07771409231737e-05, "loss_nats": 1.056404948234558, "step": 6640, "unclipped_grad_norm": 0.9801992774009705 }, { "bits_per_byte": 1.5169175935218258, "learning_rate": 5.048066138912756e-05, "loss_nats": 1.0514471530914307, "step": 6650, "unclipped_grad_norm": 0.8882671594619751 }, { "bits_per_byte": 1.5152221885496202, "learning_rate": 5.018613867460863e-05, "loss_nats": 1.050271987915039, "step": 6660, "unclipped_grad_norm": 0.9029646515846252 }, { "bits_per_byte": 1.4896082964086694, "learning_rate": 4.989357781219755e-05, "loss_nats": 1.0325177907943726, "step": 6670, "unclipped_grad_norm": 0.9797500371932983 }, { "bits_per_byte": 1.4764526555135697, "learning_rate": 4.960298380095222e-05, "loss_nats": 1.023398995399475, "step": 6680, "unclipped_grad_norm": 0.9356769919395447 }, { "bits_per_byte": 1.4958203097578342, "learning_rate": 4.931436160632256e-05, "loss_nats": 1.0368236303329468, "step": 6690, "unclipped_grad_norm": 0.939789891242981 }, { "bits_per_byte": 1.5480882451121538, "learning_rate": 4.9027716160065485e-05, "loss_nats": 1.073053002357483, "step": 6700, "unclipped_grad_norm": 1.0080333948135376 }, { "bits_per_byte": 1.525887864625528, "learning_rate": 4.874305236016084e-05, "loss_nats": 1.0576648712158203, "step": 6710, "unclipped_grad_norm": 0.9197862148284912 }, { "bits_per_byte": 1.534906462854123, "learning_rate": 4.846037507072752e-05, "loss_nats": 1.0639160871505737, "step": 6720, "unclipped_grad_norm": 0.9543269872665405 }, { "bits_per_byte": 1.4820493149379474, "learning_rate": 4.817968912194049e-05, "loss_nats": 1.0272783041000366, "step": 6730, "unclipped_grad_norm": 0.9817740321159363 }, { "bits_per_byte": 1.4384499931177168, "learning_rate": 4.790099930994813e-05, "loss_nats": 0.9970575571060181, "step": 6740, "unclipped_grad_norm": 0.9710240364074707 }, { "bits_per_byte": 1.511069151496585, "learning_rate": 4.762431039679047e-05, "loss_nats": 1.0473933219909668, "step": 6750, "unclipped_grad_norm": 0.9612263441085815 }, { "bits_per_byte": 1.4780942299160553, "learning_rate": 4.734962711031755e-05, "loss_nats": 1.0245368480682373, "step": 6760, "unclipped_grad_norm": 0.9238441586494446 }, { "bits_per_byte": 1.3767874194686909, "learning_rate": 4.707695414410892e-05, "loss_nats": 0.9543163180351257, "step": 6770, "unclipped_grad_norm": 0.9215719699859619 }, { "bits_per_byte": 1.4785716537548506, "learning_rate": 4.680629615739322e-05, "loss_nats": 1.0248677730560303, "step": 6780, "unclipped_grad_norm": 0.9403936266899109 }, { "bits_per_byte": 1.4459957319243224, "learning_rate": 4.653765777496863e-05, "loss_nats": 1.0022878646850586, "step": 6790, "unclipped_grad_norm": 0.9651554822921753 }, { "bits_per_byte": 1.5175859181030786, "learning_rate": 4.6271043587123986e-05, "loss_nats": 1.051910400390625, "step": 6800, "unclipped_grad_norm": 0.9926899075508118 }, { "bits_per_byte": 1.5110634760691066, "learning_rate": 4.600645814956006e-05, "loss_nats": 1.0473893880844116, "step": 6810, "unclipped_grad_norm": 0.9793192148208618 }, { "bits_per_byte": 1.5052492585913877, "learning_rate": 4.574390598331204e-05, "loss_nats": 1.0433592796325684, "step": 6820, "unclipped_grad_norm": 1.0081095695495605 }, { "bits_per_byte": 1.466343515295854, "learning_rate": 4.548339157467199e-05, "loss_nats": 1.0163918733596802, "step": 6830, "unclipped_grad_norm": 0.960361659526825 }, { "bits_per_byte": 1.4376153613130795, "learning_rate": 4.52249193751124e-05, "loss_nats": 0.9964790344238281, "step": 6840, "unclipped_grad_norm": 0.9702624678611755 }, { "bits_per_byte": 1.4922177891702109, "learning_rate": 4.4968493801209985e-05, "loss_nats": 1.0343265533447266, "step": 6850, "unclipped_grad_norm": 0.9893983006477356 }, { "bits_per_byte": 1.4659078832412185, "learning_rate": 4.471411923457034e-05, "loss_nats": 1.016089916229248, "step": 6860, "unclipped_grad_norm": 0.9693014621734619 }, { "bits_per_byte": 1.413224953750367, "learning_rate": 4.446180002175291e-05, "loss_nats": 0.9795728921890259, "step": 6870, "unclipped_grad_norm": 0.9252057075500488 }, { "bits_per_byte": 1.4200107012228098, "learning_rate": 4.421154047419693e-05, "loss_nats": 0.9842764139175415, "step": 6880, "unclipped_grad_norm": 0.8884434700012207 }, { "bits_per_byte": 1.4366857090938405, "learning_rate": 4.396334486814753e-05, "loss_nats": 0.9958346486091614, "step": 6890, "unclipped_grad_norm": 0.9679457545280457 }, { "bits_per_byte": 1.5416700245646047, "learning_rate": 4.371721744458285e-05, "loss_nats": 1.0686042308807373, "step": 6900, "unclipped_grad_norm": 0.9985603094100952 }, { "bits_per_byte": 1.4893800754309745, "learning_rate": 4.347316240914145e-05, "loss_nats": 1.0323596000671387, "step": 6910, "unclipped_grad_norm": 0.9936915636062622 }, { "bits_per_byte": 1.4499423897963222, "learning_rate": 4.3231183932050546e-05, "loss_nats": 1.00502347946167, "step": 6920, "unclipped_grad_norm": 0.923648476600647 }, { "bits_per_byte": 1.4406846496917027, "learning_rate": 4.299128614805464e-05, "loss_nats": 0.9986065030097961, "step": 6930, "unclipped_grad_norm": 0.9748247861862183 }, { "bits_per_byte": 1.408974058568991, "learning_rate": 4.2753473156344965e-05, "loss_nats": 0.9766263961791992, "step": 6940, "unclipped_grad_norm": 0.9357670545578003 }, { "bits_per_byte": 1.434796565665438, "learning_rate": 4.251774902048942e-05, "loss_nats": 0.9945251941680908, "step": 6950, "unclipped_grad_norm": 0.9624595642089844 }, { "bits_per_byte": 1.49675830313564, "learning_rate": 4.228411776836306e-05, "loss_nats": 1.0374737977981567, "step": 6960, "unclipped_grad_norm": 0.9387544989585876 }, { "bits_per_byte": 1.4929531869853003, "learning_rate": 4.2052583392079427e-05, "loss_nats": 1.0348362922668457, "step": 6970, "unclipped_grad_norm": 0.9343914985656738 }, { "bits_per_byte": 1.4660633555575984, "learning_rate": 4.182314984792214e-05, "loss_nats": 1.016197681427002, "step": 6980, "unclipped_grad_norm": 0.9326976537704468 }, { "bits_per_byte": 1.492015881538098, "learning_rate": 4.159582105627748e-05, "loss_nats": 1.034186601638794, "step": 6990, "unclipped_grad_norm": 1.0069916248321533 }, { "bits_per_byte": 1.52302985693349, "learning_rate": 4.137060090156724e-05, "loss_nats": 1.0556838512420654, "step": 7000, "unclipped_grad_norm": 0.9719612002372742 }, { "bits_per_byte": 1.512785538352196, "learning_rate": 4.114749323218249e-05, "loss_nats": 1.0485830307006836, "step": 7010, "unclipped_grad_norm": 0.9873834848403931 }, { "bits_per_byte": 1.4473323810868282, "learning_rate": 4.0926501860417706e-05, "loss_nats": 1.0032143592834473, "step": 7020, "unclipped_grad_norm": 0.939332127571106 }, { "bits_per_byte": 1.5166639191118032, "learning_rate": 4.070763056240571e-05, "loss_nats": 1.0512713193893433, "step": 7030, "unclipped_grad_norm": 0.9837853312492371 }, { "bits_per_byte": 1.5072451172546506, "learning_rate": 4.049088307805306e-05, "loss_nats": 1.0447427034378052, "step": 7040, "unclipped_grad_norm": 0.9734936952590942 }, { "bits_per_byte": 1.4796651194490362, "learning_rate": 4.027626311097629e-05, "loss_nats": 1.0256257057189941, "step": 7050, "unclipped_grad_norm": 1.005554437637329 }, { "bits_per_byte": 1.4927048440374544, "learning_rate": 4.006377432843846e-05, "loss_nats": 1.0346641540527344, "step": 7060, "unclipped_grad_norm": 0.9903765320777893 }, { "bits_per_byte": 1.4561590466770602, "learning_rate": 3.985342036128656e-05, "loss_nats": 1.009332537651062, "step": 7070, "unclipped_grad_norm": 1.009922981262207 }, { "bits_per_byte": 1.4353391709289103, "learning_rate": 3.964520480388956e-05, "loss_nats": 0.9949012994766235, "step": 7080, "unclipped_grad_norm": 0.9552373290061951 }, { "bits_per_byte": 1.4780041110070032, "learning_rate": 3.943913121407678e-05, "loss_nats": 1.0244743824005127, "step": 7090, "unclipped_grad_norm": 1.0046122074127197 }, { "bits_per_byte": 1.4728850474040718, "learning_rate": 3.923520311307735e-05, "loss_nats": 1.0209261178970337, "step": 7100, "unclipped_grad_norm": 0.9868813157081604 }, { "bits_per_byte": 1.4862149067244954, "learning_rate": 3.9033423985459816e-05, "loss_nats": 1.030165672302246, "step": 7110, "unclipped_grad_norm": 0.9593554735183716 }, { "bits_per_byte": 1.5328834309320238, "learning_rate": 3.88337972790728e-05, "loss_nats": 1.062513828277588, "step": 7120, "unclipped_grad_norm": 1.0129024982452393 }, { "bits_per_byte": 1.4811555211014134, "learning_rate": 3.8636326404985845e-05, "loss_nats": 1.0266587734222412, "step": 7130, "unclipped_grad_norm": 1.0546875 }, { "bits_per_byte": 1.4724865636020226, "learning_rate": 3.844101473743143e-05, "loss_nats": 1.0206499099731445, "step": 7140, "unclipped_grad_norm": 0.9776664972305298 }, { "bits_per_byte": 1.4357171887955067, "learning_rate": 3.824786561374701e-05, "loss_nats": 0.9951633214950562, "step": 7150, "unclipped_grad_norm": 0.9291411638259888 }, { "bits_per_byte": 1.5232358921492237, "learning_rate": 3.805688233431824e-05, "loss_nats": 1.0558266639709473, "step": 7160, "unclipped_grad_norm": 1.0189123153686523 }, { "bits_per_byte": 1.4618840051589814, "learning_rate": 3.786806816252236e-05, "loss_nats": 1.0133007764816284, "step": 7170, "unclipped_grad_norm": 0.9592419266700745 }, { "bits_per_byte": 1.4762450724539784, "learning_rate": 3.768142632467269e-05, "loss_nats": 1.0232551097869873, "step": 7180, "unclipped_grad_norm": 0.9680334329605103 }, { "bits_per_byte": 1.4441101141402626, "learning_rate": 3.7496960009963236e-05, "loss_nats": 1.0009808540344238, "step": 7190, "unclipped_grad_norm": 1.0148857831954956 }, { "bits_per_byte": 1.472608843266786, "learning_rate": 3.7314672370414325e-05, "loss_nats": 1.0207346677780151, "step": 7200, "unclipped_grad_norm": 0.9935786128044128 }, { "bits_per_byte": 1.5299951542934338, "learning_rate": 3.713456652081882e-05, "loss_nats": 1.060511827468872, "step": 7210, "unclipped_grad_norm": 0.9725626111030579 }, { "bits_per_byte": 1.5238232129019202, "learning_rate": 3.695664553868873e-05, "loss_nats": 1.0562337636947632, "step": 7220, "unclipped_grad_norm": 1.0141370296478271 }, { "bits_per_byte": 1.4390708504873315, "learning_rate": 3.678091246420277e-05, "loss_nats": 0.9974879026412964, "step": 7230, "unclipped_grad_norm": 1.0297694206237793 }, { "bits_per_byte": 1.4632025961431465, "learning_rate": 3.660737030015427e-05, "loss_nats": 1.0142147541046143, "step": 7240, "unclipped_grad_norm": 0.9345177412033081 }, { "bits_per_byte": 1.5031187375124992, "learning_rate": 3.6436022011900056e-05, "loss_nats": 1.0418825149536133, "step": 7250, "unclipped_grad_norm": 1.0408201217651367 }, { "bits_per_byte": 1.3820238612308362, "learning_rate": 3.626687052730954e-05, "loss_nats": 0.9579459428787231, "step": 7260, "unclipped_grad_norm": 0.9544695615768433 }, { "bits_per_byte": 1.4274823155068936, "learning_rate": 3.6099918736714955e-05, "loss_nats": 0.9894553422927856, "step": 7270, "unclipped_grad_norm": 0.9940797686576843 }, { "bits_per_byte": 1.4990891840027836, "learning_rate": 3.5935169492861716e-05, "loss_nats": 1.0390894412994385, "step": 7280, "unclipped_grad_norm": 1.013985514640808 }, { "bits_per_byte": 1.4370805812602216, "learning_rate": 3.5772625610859886e-05, "loss_nats": 0.99610835313797, "step": 7290, "unclipped_grad_norm": 0.9820091724395752 }, { "bits_per_byte": 1.4451605841717328, "learning_rate": 3.5612289868135887e-05, "loss_nats": 1.001708984375, "step": 7300, "unclipped_grad_norm": 0.9996907114982605 }, { "bits_per_byte": 1.4502488628801598, "learning_rate": 3.545416500438522e-05, "loss_nats": 1.0052359104156494, "step": 7310, "unclipped_grad_norm": 0.9723894000053406 }, { "bits_per_byte": 1.4141343980081293, "learning_rate": 3.52982537215255e-05, "loss_nats": 0.9802032709121704, "step": 7320, "unclipped_grad_norm": 0.9815637469291687 }, { "bits_per_byte": 1.429772436485783, "learning_rate": 3.5144558683650396e-05, "loss_nats": 0.9910427331924438, "step": 7330, "unclipped_grad_norm": 1.0089058876037598 }, { "bits_per_byte": 1.4033304478809279, "learning_rate": 3.499308251698404e-05, "loss_nats": 0.9727145433425903, "step": 7340, "unclipped_grad_norm": 0.9830491542816162 }, { "bits_per_byte": 1.4800763299672492, "learning_rate": 3.484382780983616e-05, "loss_nats": 1.02591073513031, "step": 7350, "unclipped_grad_norm": 0.9842677712440491 }, { "bits_per_byte": 1.4409101189469837, "learning_rate": 3.469679711255795e-05, "loss_nats": 0.9987627863883972, "step": 7360, "unclipped_grad_norm": 1.0266367197036743 }, { "bits_per_byte": 1.4700960047550296, "learning_rate": 3.4551992937498284e-05, "loss_nats": 1.0189929008483887, "step": 7370, "unclipped_grad_norm": 0.9935671091079712 }, { "bits_per_byte": 1.4111584102175994, "learning_rate": 3.440941775896104e-05, "loss_nats": 0.9781404733657837, "step": 7380, "unclipped_grad_norm": 0.9791932702064514 }, { "bits_per_byte": 1.4805810990475317, "learning_rate": 3.4269074013162596e-05, "loss_nats": 1.0262606143951416, "step": 7390, "unclipped_grad_norm": 0.9806758761405945 }, { "bits_per_byte": 1.422500752033327, "learning_rate": 3.41309640981904e-05, "loss_nats": 0.9860023856163025, "step": 7400, "unclipped_grad_norm": 1.014562726020813 }, { "bits_per_byte": 1.419007526420327, "learning_rate": 3.3995090373961795e-05, "loss_nats": 0.9835810661315918, "step": 7410, "unclipped_grad_norm": 0.9609370231628418 }, { "bits_per_byte": 1.483305992167802, "learning_rate": 3.3861455162183876e-05, "loss_nats": 1.0281493663787842, "step": 7420, "unclipped_grad_norm": 0.9924190640449524 }, { "bits_per_byte": 1.4892452410326982, "learning_rate": 3.3730060746313664e-05, "loss_nats": 1.0322661399841309, "step": 7430, "unclipped_grad_norm": 1.0244429111480713 }, { "bits_per_byte": 1.50888548777858, "learning_rate": 3.3600909371519225e-05, "loss_nats": 1.0458797216415405, "step": 7440, "unclipped_grad_norm": 1.0394790172576904 }, { "bits_per_byte": 1.4553271665948369, "learning_rate": 3.3474003244641194e-05, "loss_nats": 1.0087559223175049, "step": 7450, "unclipped_grad_norm": 0.99248868227005 }, { "bits_per_byte": 1.4425173828106221, "learning_rate": 3.334934453415514e-05, "loss_nats": 0.999876856803894, "step": 7460, "unclipped_grad_norm": 1.0004469156265259 }, { "bits_per_byte": 1.437790267669007, "learning_rate": 3.3226935370134486e-05, "loss_nats": 0.9966002702713013, "step": 7470, "unclipped_grad_norm": 1.0109968185424805 }, { "bits_per_byte": 1.4682612938356252, "learning_rate": 3.31067778442141e-05, "loss_nats": 1.017721176147461, "step": 7480, "unclipped_grad_norm": 1.0299010276794434 }, { "bits_per_byte": 1.5222762289574092, "learning_rate": 3.298887400955457e-05, "loss_nats": 1.055161476135254, "step": 7490, "unclipped_grad_norm": 1.0327476263046265 }, { "bits_per_byte": 1.4747042798848986, "learning_rate": 3.287322588080714e-05, "loss_nats": 1.0221871137619019, "step": 7500, "unclipped_grad_norm": 0.9886637926101685 }, { "bits_per_byte": 1.4352613487793948, "learning_rate": 3.275983543407924e-05, "loss_nats": 0.9948473572731018, "step": 7510, "unclipped_grad_norm": 0.9710773825645447 }, { "bits_per_byte": 1.4570311706995855, "learning_rate": 3.2648704606900735e-05, "loss_nats": 1.009937047958374, "step": 7520, "unclipped_grad_norm": 1.0144232511520386 }, { "bits_per_byte": 1.502727820947088, "learning_rate": 3.253983529819086e-05, "loss_nats": 1.0416115522384644, "step": 7530, "unclipped_grad_norm": 1.0307852029800415 }, { "bits_per_byte": 1.4572943041554056, "learning_rate": 3.243322936822571e-05, "loss_nats": 1.0101194381713867, "step": 7540, "unclipped_grad_norm": 0.9732069969177246 }, { "bits_per_byte": 1.4747278415080667, "learning_rate": 3.232888863860648e-05, "loss_nats": 1.0222034454345703, "step": 7550, "unclipped_grad_norm": 1.0250486135482788 }, { "bits_per_byte": 1.4513507557242378, "learning_rate": 3.222681489222838e-05, "loss_nats": 1.0059996843338013, "step": 7560, "unclipped_grad_norm": 1.0031380653381348 }, { "bits_per_byte": 1.4127096077370564, "learning_rate": 3.212700987325009e-05, "loss_nats": 0.979215681552887, "step": 7570, "unclipped_grad_norm": 0.9720339775085449 }, { "bits_per_byte": 1.587541409185384, "learning_rate": 3.202947528706403e-05, "loss_nats": 1.1003998517990112, "step": 7580, "unclipped_grad_norm": 1.080464482307434 }, { "bits_per_byte": 1.4421111597892842, "learning_rate": 3.1934212800267116e-05, "loss_nats": 0.9995952844619751, "step": 7590, "unclipped_grad_norm": 0.9815735220909119 }, { "bits_per_byte": 1.4267507013101233, "learning_rate": 3.1841224040632484e-05, "loss_nats": 0.9889482259750366, "step": 7600, "unclipped_grad_norm": 1.029758334159851 }, { "bits_per_byte": 1.4539179407536669, "learning_rate": 3.175051059708143e-05, "loss_nats": 1.0077791213989258, "step": 7610, "unclipped_grad_norm": 1.031221628189087 }, { "bits_per_byte": 1.4028547438686412, "learning_rate": 3.1662074019656414e-05, "loss_nats": 0.9723848104476929, "step": 7620, "unclipped_grad_norm": 0.9562389850616455 }, { "bits_per_byte": 1.3938100042771153, "learning_rate": 3.157591581949455e-05, "loss_nats": 0.9661154747009277, "step": 7630, "unclipped_grad_norm": 1.023301124572754 }, { "bits_per_byte": 1.4184527103886437, "learning_rate": 3.149203746880175e-05, "loss_nats": 0.983196496963501, "step": 7640, "unclipped_grad_norm": 0.9981763958930969 }, { "bits_per_byte": 1.5608753271864544, "learning_rate": 3.141044040082763e-05, "loss_nats": 1.081916332244873, "step": 7650, "unclipped_grad_norm": 1.0295648574829102 }, { "bits_per_byte": 1.4665378556913289, "learning_rate": 3.133112600984091e-05, "loss_nats": 1.0165265798568726, "step": 7660, "unclipped_grad_norm": 1.0587892532348633 }, { "bits_per_byte": 1.4931012640476933, "learning_rate": 3.125409565110571e-05, "loss_nats": 1.034938931465149, "step": 7670, "unclipped_grad_norm": 1.0342050790786743 }, { "bits_per_byte": 1.4465245785757257, "learning_rate": 3.117935064085835e-05, "loss_nats": 1.0026544332504272, "step": 7680, "unclipped_grad_norm": 1.0567405223846436 }, { "bits_per_byte": 1.4541416901824393, "learning_rate": 3.110689225628479e-05, "loss_nats": 1.0079342126846313, "step": 7690, "unclipped_grad_norm": 1.0190099477767944 }, { "bits_per_byte": 1.3949941048283057, "learning_rate": 3.1036721735498926e-05, "loss_nats": 0.9669362306594849, "step": 7700, "unclipped_grad_norm": 1.002891182899475 }, { "bits_per_byte": 1.4762163513512843, "learning_rate": 3.096884027752132e-05, "loss_nats": 1.0232352018356323, "step": 7710, "unclipped_grad_norm": 0.9984713792800903 }, { "bits_per_byte": 1.480895139368007, "learning_rate": 3.090324904225885e-05, "loss_nats": 1.0264782905578613, "step": 7720, "unclipped_grad_norm": 1.0342158079147339 }, { "bits_per_byte": 1.486845911070511, "learning_rate": 3.083994915048472e-05, "loss_nats": 1.030603051185608, "step": 7730, "unclipped_grad_norm": 1.027107834815979 }, { "bits_per_byte": 1.4387832095037634, "learning_rate": 3.077894168381942e-05, "loss_nats": 0.9972885251045227, "step": 7740, "unclipped_grad_norm": 1.0455139875411987 }, { "bits_per_byte": 1.4381068877292456, "learning_rate": 3.0720227684712254e-05, "loss_nats": 0.9968197345733643, "step": 7750, "unclipped_grad_norm": 1.0159157514572144 }, { "bits_per_byte": 1.4538223183997872, "learning_rate": 3.0663808156423456e-05, "loss_nats": 1.0077128410339355, "step": 7760, "unclipped_grad_norm": 0.9867706894874573 }, { "bits_per_byte": 1.3986834766546148, "learning_rate": 3.060968406300711e-05, "loss_nats": 0.9694935083389282, "step": 7770, "unclipped_grad_norm": 0.9943742156028748 }, { "bits_per_byte": 1.4760859885019304, "learning_rate": 3.055785632929463e-05, "loss_nats": 1.0231448411941528, "step": 7780, "unclipped_grad_norm": 1.0422483682632446 }, { "bits_per_byte": 1.4162891686407895, "learning_rate": 3.0508325840878984e-05, "loss_nats": 0.9816968441009521, "step": 7790, "unclipped_grad_norm": 0.9937946796417236 }, { "bits_per_byte": 1.45477768202533, "learning_rate": 3.046109344409957e-05, "loss_nats": 1.0083750486373901, "step": 7800, "unclipped_grad_norm": 1.0428839921951294 }, { "bits_per_byte": 1.4292601001688625, "learning_rate": 3.0416159946027727e-05, "loss_nats": 0.9906876087188721, "step": 7810, "unclipped_grad_norm": 1.028254508972168 }, { "bits_per_byte": 1.4070698666586379, "learning_rate": 3.0373526114452946e-05, "loss_nats": 0.975306510925293, "step": 7820, "unclipped_grad_norm": 1.0253064632415771 }, { "bits_per_byte": 1.4530960356651936, "learning_rate": 3.0333192677869803e-05, "loss_nats": 1.0072094202041626, "step": 7830, "unclipped_grad_norm": 1.0885143280029297 }, { "bits_per_byte": 1.4330722676078875, "learning_rate": 3.029516032546544e-05, "loss_nats": 0.9933300018310547, "step": 7840, "unclipped_grad_norm": 1.0074925422668457 }, { "bits_per_byte": 1.3858685333908742, "learning_rate": 3.02594297071078e-05, "loss_nats": 0.9606108665466309, "step": 7850, "unclipped_grad_norm": 0.9880329966545105 }, { "bits_per_byte": 1.4517502714221922, "learning_rate": 3.0226001433334603e-05, "loss_nats": 1.0062766075134277, "step": 7860, "unclipped_grad_norm": 1.0082415342330933 }, { "bits_per_byte": 1.4675547891058807, "learning_rate": 3.019487607534278e-05, "loss_nats": 1.0172314643859863, "step": 7870, "unclipped_grad_norm": 1.023923397064209 }, { "bits_per_byte": 1.3867994754799946, "learning_rate": 3.016605416497886e-05, "loss_nats": 0.9612561464309692, "step": 7880, "unclipped_grad_norm": 1.0151392221450806 }, { "bits_per_byte": 1.4568015738606837, "learning_rate": 3.0139536194729734e-05, "loss_nats": 1.0097779035568237, "step": 7890, "unclipped_grad_norm": 1.0505098104476929 }, { "bits_per_byte": 1.446776189193938, "learning_rate": 3.0115322617714362e-05, "loss_nats": 1.00282883644104, "step": 7900, "unclipped_grad_norm": 1.0279992818832397 }, { "bits_per_byte": 1.461830518554563, "learning_rate": 3.0093413847675965e-05, "loss_nats": 1.0132637023925781, "step": 7910, "unclipped_grad_norm": 1.0125948190689087 }, { "bits_per_byte": 1.4593548282953928, "learning_rate": 3.0073810258974985e-05, "loss_nats": 1.0115476846694946, "step": 7920, "unclipped_grad_norm": 1.0411789417266846 }, { "bits_per_byte": 1.4317973602170415, "learning_rate": 3.0056512186582658e-05, "loss_nats": 0.9924463033676147, "step": 7930, "unclipped_grad_norm": 1.0330348014831543 }, { "bits_per_byte": 1.431196452834927, "learning_rate": 3.0041519926075316e-05, "loss_nats": 0.9920297861099243, "step": 7940, "unclipped_grad_norm": 1.0142537355422974 }, { "bits_per_byte": 1.49434246283897, "learning_rate": 3.0028833733629313e-05, "loss_nats": 1.035799264907837, "step": 7950, "unclipped_grad_norm": 1.0716009140014648 }, { "bits_per_byte": 1.4186909923514142, "learning_rate": 3.0018453826016686e-05, "loss_nats": 0.9833616614341736, "step": 7960, "unclipped_grad_norm": 1.0115852355957031 }, { "bits_per_byte": 1.4420433126335188, "learning_rate": 3.001038038060143e-05, "loss_nats": 0.9995482563972473, "step": 7970, "unclipped_grad_norm": 1.0335451364517212 }, { "bits_per_byte": 1.4872165336831205, "learning_rate": 3.0004613535336433e-05, "loss_nats": 1.0308599472045898, "step": 7980, "unclipped_grad_norm": 1.0748283863067627 }, { "bits_per_byte": 1.5234063269562288, "learning_rate": 3.000115338876117e-05, "loss_nats": 1.055944800376892, "step": 7990, "unclipped_grad_norm": 1.076164722442627 }, { "bits_per_byte": 1.4309937712808851, "learning_rate": 3e-05, "loss_nats": 0.9918892979621887, "step": 8000, "unclipped_grad_norm": 1.0322442054748535 } ] }